Improve configure/release script to work with pinned requirements

.travis.yml

@@ -1,6 +1,6 @@
 
 ################################################################################
-# We use Travis to run minimal smoke tests suites on Pythons 2.7 and 3.6
+# We use Travis to run minimal smoke tests suites on Pythons 3.6
 # on macOS and Linux
 # We also run the documentation build and the ABOUT file checks
 ################################################################################
@@ -9,32 +9,11 @@ matrix:
   include:
     # Run minimal test suite
     - os: osx
-      env: 
-        - PYTHON="3.6.8"
-        - TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
+      osx_image: xcode10
       language: generic
-      python:
-
-    - os: osx
       env: 
-        - PYTHON="2.7.15"
+        - PYTHON=36
         - TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
-      language: generic
-      python:
-
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
-      language: python
-      python: "2.7"
-      dist: xenial
-
-    - os: linux
-      sudo: required
-      env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode"
-      language: python
-      python: "2.7"
-      dist: bionic
 
     - os: linux
       sudo: required
@@ -93,8 +72,20 @@ before_install:
   - chmod +x ./docs/scripts/sphinx_build_link_check.sh
   - chmod +x ./docs/scripts/doc8_style_check.sh
 
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    curl -LO https://raw.githubusercontent.com/nexB/macports-ci/master/macports-ci;
+    source ./macports-ci install;
+    fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    yes | sudo port install python$PYTHON;
+    yes | sudo port install py$PYTHON-pip;
+    sudo port select --set python3 python$PYTHON;
+    sudo port select --set pip pip$PYTHON;
+    export PATH=$PATH:/Users/travis/Library/Python/$PYTHON/bin;
+    fi
+
 install:
-  - ./configure
+  - PYTHON_EXE=python3 ./configure
 
 script:
   # If debugging, use a subset of tests to wait less:

appveyor.yml

@@ -5,8 +5,6 @@
 ################################################################################
 environment:
     matrix:
-        - PYTHON: "C:\\Python27"
-        - PYTHON: "C:\\Python27-x64"
         - PYTHON: "C:\\Python36"
         - PYTHON: "C:\\Python36-x64"
 #        - PYTHON: "C:\\Python37"

azure-pipelines.yml

@@ -1,193 +1,11 @@
 
 ################################################################################
-# We use Azure to run the full tests suites on Pythons 2.7 and 3.6
+# We use Azure to run the full tests suites on Pythons 3.6 and above.
 # on Windows (32 and 64), macOS and Linux (64 various distro)
 ################################################################################
 
 jobs:
 
-################################################################################
-# These jobs are using VMs and Azure-provided Pythons 2.7
-################################################################################
-
-    - template: etc/ci/azure-linux.yml
-      parameters:
-          job_name: vm_ubuntu16_py27
-          image_name: ubuntu-16.04
-          python_versions: ['2.7']
-          test_suites:
-              misc: |
-                bin/py.test -n 2 -vvs --reruns=3 --test-suite=all \
-                  --ignore=tests/scancode \
-                  --ignore=tests/cluecode \
-                  --ignore=tests/licensedcode \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              scancode: |
-                bin/py.test --reruns=3 -vvs --test-suite=all \
-                  tests/scancode \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              cluecode: |
-                bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \
-                  tests/cluecode \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              license_base: |
-                bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \
-                  --ignore=tests/licensedcode/test_zzzz_cache.py \
-                  --ignore=tests/licensedcode/test_detection_datadriven1.py \
-                  --ignore=tests/licensedcode/test_detection_datadriven2.py \
-                  tests/licensedcode \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              license_datadriven1: |
-                bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \
-                  tests/licensedcode/test_detection_datadriven1.py \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              license_datadriven2: |
-                bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \
-                  tests/licensedcode/test_detection_datadriven2.py \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-              license_cache: |
-                bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \
-                  tests/licensedcode/test_zzzz_cache.py \
-                  --cov=src --cov-report=term --cov-report=xml
-                bin/codecov --token "$CODECOV_TOKEN"
-
-    - template: etc/ci/azure-win.yml
-      parameters:
-          job_name: Win2016_32_py27
-          image_name: vs2017-win2016
-          python_versions: ['2.7']
-          python_architecture: x86
-          test_suites:
-              misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode  --ignore=tests\licensedcode
-              scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
-              lic_cluecode: Scripts\py.test -vvs  --reruns=3 tests\cluecode  tests\licensedcode
-
-    - template: etc/ci/azure-win.yml
-      parameters:
-          job_name: Win2016_64_py27
-          image_name: vs2017-win2016
-          python_versions: ['2.7']
-          python_architecture: x64
-          test_suites:
-              misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode  --ignore=tests\licensedcode
-              scancode: Scripts\py.test -vvs --reruns=3 tests\scancode
-              lic_cluecode: Scripts\py.test -vvs  --reruns=3 tests\cluecode  tests\licensedcode
-
-    - template: etc/ci/azure-mac.yml
-      parameters:
-          job_name: macos1015_py27
-          image_name: macos-10.15
-          python_versions: ['2.7']
-          test_suites:
-              all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode
-              scancode: bin/py.test -vvs --reruns=3 tests/scancode
-
-
-################################################################################
-# These jobs are using containers and their own Python 2.7
-################################################################################
-
-    - template: etc/ci/azure-container-deb.yml
-      parameters:
-          job_name: ubuntu18_py27
-          container: 'ubuntu:bionic'
-          python_path: python2.7
-          python_version: '2.7'
-          install_python: sudo apt-get install -y python python-dev
-          test_suite_label: all
-          test_suite: bin/py.test -n 2 -vvs --reruns=3
-
-#    - template: etc/ci/azure-container-rpm.yml
-#      parameters:
-#          job_name: centos7_py27
-#          container: centos:7
-#          python_path: python2.7
-#          python_version: '2.7'
-#          install_python: sudo yum install -y python python27-devel
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-#
-#    - template: etc/ci/azure-container-rpm.yml
-#      parameters:
-#          job_name: manylinux1_py27
-#          container: quay.io/pypa/manylinux1_x86_64:latest
-#          python_path: /opt/python/cp27-cp27mu/bin/python
-#          python_version: '2.7'
-#          install_packages: echo "No extra packages for now"
-#          install_python: echo "Python is pre-installed"
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-#
-#    - template: etc/ci/azure-container-rpm.yml
-#      parameters:
-#          job_name: manylinux2010_py27
-#          container: quay.io/pypa/manylinux2010_x86_64:latest
-#          python_path: /opt/python/cp27-cp27mu/bin/python
-#          python_version: '2.7'
-#          install_packages: echo "No extra packages for now"
-#          install_python: echo "Python is pre-installed"
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-#
-#    - template: etc/ci/azure-container-rpm.yml
-#      parameters:
-#          job_name: fedora30_py27
-#          container: fedora:30
-#          package_manager: dnf
-#          python_path: python2.7
-#          python_version: '2.7'
-#          install_python: |
-#              set -e -x
-#              sudo dnf install -y python2 python2-devel
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-#
-#    - template: etc/ci/azure-container-deb.yml
-#      parameters:
-#          job_name: debian9_py27
-#          container: debian:stretch
-#          python_path: python2.7
-#          python_version: '2.7'
-#          install_python: sudo apt-get install -y python python-dev
-#          test_suite_label: all
-#          test_suite: bin/py.test -n 2 -vvs --reruns=3
-
-
-########################################################################
-# RELEASE on 2.7. Also check that we can pip install
-########################################################################
-
-    - job: Build_release_archive_py2
-      pool:
-          vmImage: ubuntu-16.04
-      steps:
-          - checkout: self
-            fetchDepth: 10
-
-          - task: UsePythonVersion@0
-            inputs:
-                versionSpec: '2.7'
-            displayName: 'Install Python 2.7'
-
-          - script: ./etc/release/release.sh
-            displayName: 'Build installable releases'
-
-          - script: ./etc/release/pip-install2.sh
-            displayName: 'Test pip installation'
-
-
 ########################################################################
 # RELEASE on 3.6. Also check that we can pip install
 ########################################################################

configure

@@ -19,10 +19,11 @@ fi
 export TPP_DIR="thirdparty"
 
 # default configurations for dev
-CONF_DEFAULT="etc/conf/dev"
+CONF_DEFAULT="etc/conf" 
 
-if [[ "$CONF_DEFAULT" == "etc/conf/dev" ]]; then
-    export TPP_DIR_DEV="thirdparty/dev"
+if [[ "$CONF_DEFAULT" == "etc/conf" ]]; then
+    #export TPP_DIR_DEV="thirdparty/dev"
+    export TPP_DIR_links="https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0"
 fi
 
 

configure.bat

@@ -26,11 +26,12 @@
 set "TPP_DIR=thirdparty"
 
 @rem # default configurations for dev
-set "CONF_DEFAULT=etc/conf/dev"
+set "CONF_DEFAULT=etc/conf"
 
 @rem # default thirdparty dist for dev
-if ""%CONF_DEFAULT%""==""etc/conf/dev"" (
-    set "TPP_DIR_DEV=thirdparty/dev"
+if ""%CONF_DEFAULT%""==""etc/conf"" (
+    @rem # set "TPP_DIR_DEV=thirdparty/dev"
+    set "TPP_DIR_links=https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0"
 )
 
 @rem # default supported version for Python 3