diff --git a/.travis.yml b/.travis.yml index 3ee14b40503..c13c2ec2218 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ ################################################################################ -# We use Travis to run minimal smoke tests suites on Pythons 2.7 and 3.6 +# We use Travis to run minimal smoke tests suites on Pythons 3.6 # on macOS and Linux # We also run the documentation build and the ABOUT file checks ################################################################################ @@ -9,32 +9,11 @@ matrix: include: # Run minimal test suite - os: osx - env: - - PYTHON="3.6.8" - - TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode" + osx_image: xcode10 language: generic - python: - - - os: osx env: - - PYTHON="2.7.15" + - PYTHON=36 - TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode" - language: generic - python: - - - os: linux - sudo: required - env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode" - language: python - python: "2.7" - dist: xenial - - - os: linux - sudo: required - env: TEST_SUITE="bin/py.test -vvs --reruns 3 tests/scancode" - language: python - python: "2.7" - dist: bionic - os: linux sudo: required @@ -93,8 +72,20 @@ before_install: - chmod +x ./docs/scripts/sphinx_build_link_check.sh - chmod +x ./docs/scripts/doc8_style_check.sh + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + curl -LO https://raw.githubusercontent.com/nexB/macports-ci/master/macports-ci; + source ./macports-ci install; + fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + yes | sudo port install python$PYTHON; + yes | sudo port install py$PYTHON-pip; + sudo port select --set python3 python$PYTHON; + sudo port select --set pip pip$PYTHON; + export PATH=$PATH:/Users/travis/Library/Python/$PYTHON/bin; + fi + install: - - ./configure + - PYTHON_EXE=python3 ./configure script: # If debugging, use a subset of tests to wait less: diff --git a/appveyor.yml b/appveyor.yml index 1dadef43355..c309fb6dad4 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,8 +5,6 @@ ################################################################################ environment: matrix: - - PYTHON: "C:\\Python27" - - PYTHON: "C:\\Python27-x64" - PYTHON: "C:\\Python36" - PYTHON: "C:\\Python36-x64" # - PYTHON: "C:\\Python37" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d21feaf7c9f..739eb118518 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,193 +1,11 @@ ################################################################################ -# We use Azure to run the full tests suites on Pythons 2.7 and 3.6 +# We use Azure to run the full tests suites on Pythons 3.6 and above. # on Windows (32 and 64), macOS and Linux (64 various distro) ################################################################################ jobs: -################################################################################ -# These jobs are using VMs and Azure-provided Pythons 2.7 -################################################################################ - - - template: etc/ci/azure-linux.yml - parameters: - job_name: vm_ubuntu16_py27 - image_name: ubuntu-16.04 - python_versions: ['2.7'] - test_suites: - misc: | - bin/py.test -n 2 -vvs --reruns=3 --test-suite=all \ - --ignore=tests/scancode \ - --ignore=tests/cluecode \ - --ignore=tests/licensedcode \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - scancode: | - bin/py.test --reruns=3 -vvs --test-suite=all \ - tests/scancode \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - cluecode: | - bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/cluecode \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - license_base: | - bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - --ignore=tests/licensedcode/test_zzzz_cache.py \ - --ignore=tests/licensedcode/test_detection_datadriven1.py \ - --ignore=tests/licensedcode/test_detection_datadriven2.py \ - tests/licensedcode \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - license_datadriven1: | - bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/licensedcode/test_detection_datadriven1.py \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - license_datadriven2: | - bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/licensedcode/test_detection_datadriven2.py \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - license_cache: | - bin/py.test -n 2 --reruns=3 -vvs --test-suite=all \ - tests/licensedcode/test_zzzz_cache.py \ - --cov=src --cov-report=term --cov-report=xml - bin/codecov --token "$CODECOV_TOKEN" - - - template: etc/ci/azure-win.yml - parameters: - job_name: Win2016_32_py27 - image_name: vs2017-win2016 - python_versions: ['2.7'] - python_architecture: x86 - test_suites: - misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode --ignore=tests\licensedcode - scancode: Scripts\py.test -vvs --reruns=3 tests\scancode - lic_cluecode: Scripts\py.test -vvs --reruns=3 tests\cluecode tests\licensedcode - - - template: etc/ci/azure-win.yml - parameters: - job_name: Win2016_64_py27 - image_name: vs2017-win2016 - python_versions: ['2.7'] - python_architecture: x64 - test_suites: - misc: Scripts\py.test -vvs --reruns=3 --ignore=tests\scancode --ignore=tests\cluecode --ignore=tests\licensedcode - scancode: Scripts\py.test -vvs --reruns=3 tests\scancode - lic_cluecode: Scripts\py.test -vvs --reruns=3 tests\cluecode tests\licensedcode - - - template: etc/ci/azure-mac.yml - parameters: - job_name: macos1015_py27 - image_name: macos-10.15 - python_versions: ['2.7'] - test_suites: - all: bin/py.test -n 2 -vvs --reruns=3 --ignore=tests/scancode - scancode: bin/py.test -vvs --reruns=3 tests/scancode - - -################################################################################ -# These jobs are using containers and their own Python 2.7 -################################################################################ - - - template: etc/ci/azure-container-deb.yml - parameters: - job_name: ubuntu18_py27 - container: 'ubuntu:bionic' - python_path: python2.7 - python_version: '2.7' - install_python: sudo apt-get install -y python python-dev - test_suite_label: all - test_suite: bin/py.test -n 2 -vvs --reruns=3 - -# - template: etc/ci/azure-container-rpm.yml -# parameters: -# job_name: centos7_py27 -# container: centos:7 -# python_path: python2.7 -# python_version: '2.7' -# install_python: sudo yum install -y python python27-devel -# test_suite_label: all -# test_suite: bin/py.test -n 2 -vvs --reruns=3 -# -# - template: etc/ci/azure-container-rpm.yml -# parameters: -# job_name: manylinux1_py27 -# container: quay.io/pypa/manylinux1_x86_64:latest -# python_path: /opt/python/cp27-cp27mu/bin/python -# python_version: '2.7' -# install_packages: echo "No extra packages for now" -# install_python: echo "Python is pre-installed" -# test_suite_label: all -# test_suite: bin/py.test -n 2 -vvs --reruns=3 -# -# - template: etc/ci/azure-container-rpm.yml -# parameters: -# job_name: manylinux2010_py27 -# container: quay.io/pypa/manylinux2010_x86_64:latest -# python_path: /opt/python/cp27-cp27mu/bin/python -# python_version: '2.7' -# install_packages: echo "No extra packages for now" -# install_python: echo "Python is pre-installed" -# test_suite_label: all -# test_suite: bin/py.test -n 2 -vvs --reruns=3 -# -# - template: etc/ci/azure-container-rpm.yml -# parameters: -# job_name: fedora30_py27 -# container: fedora:30 -# package_manager: dnf -# python_path: python2.7 -# python_version: '2.7' -# install_python: | -# set -e -x -# sudo dnf install -y python2 python2-devel -# test_suite_label: all -# test_suite: bin/py.test -n 2 -vvs --reruns=3 -# -# - template: etc/ci/azure-container-deb.yml -# parameters: -# job_name: debian9_py27 -# container: debian:stretch -# python_path: python2.7 -# python_version: '2.7' -# install_python: sudo apt-get install -y python python-dev -# test_suite_label: all -# test_suite: bin/py.test -n 2 -vvs --reruns=3 - - -######################################################################## -# RELEASE on 2.7. Also check that we can pip install -######################################################################## - - - job: Build_release_archive_py2 - pool: - vmImage: ubuntu-16.04 - steps: - - checkout: self - fetchDepth: 10 - - - task: UsePythonVersion@0 - inputs: - versionSpec: '2.7' - displayName: 'Install Python 2.7' - - - script: ./etc/release/release.sh - displayName: 'Build installable releases' - - - script: ./etc/release/pip-install2.sh - displayName: 'Test pip installation' - - ######################################################################## # RELEASE on 3.6. Also check that we can pip install ######################################################################## diff --git a/configure b/configure index 4fa8f858bc4..e17a7aca9d0 100755 --- a/configure +++ b/configure @@ -19,10 +19,11 @@ fi export TPP_DIR="thirdparty" # default configurations for dev -CONF_DEFAULT="etc/conf/dev" +CONF_DEFAULT="etc/conf" -if [[ "$CONF_DEFAULT" == "etc/conf/dev" ]]; then - export TPP_DIR_DEV="thirdparty/dev" +if [[ "$CONF_DEFAULT" == "etc/conf" ]]; then + #export TPP_DIR_DEV="thirdparty/dev" + export TPP_DIR_links="https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0" fi diff --git a/configure.bat b/configure.bat index 4755018bc49..9fc322d1cee 100644 --- a/configure.bat +++ b/configure.bat @@ -26,11 +26,12 @@ set "TPP_DIR=thirdparty" @rem # default configurations for dev -set "CONF_DEFAULT=etc/conf/dev" +set "CONF_DEFAULT=etc/conf" @rem # default thirdparty dist for dev -if ""%CONF_DEFAULT%""==""etc/conf/dev"" ( - set "TPP_DIR_DEV=thirdparty/dev" +if ""%CONF_DEFAULT%""==""etc/conf"" ( + @rem # set "TPP_DIR_DEV=thirdparty/dev" + set "TPP_DIR_links=https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0" ) @rem # default supported version for Python 3 diff --git a/etc/conf/dev/requirements_base.txt b/etc/conf/dev/requirements_base.txt index 672b35497ef..f5ef622905d 100644 --- a/etc/conf/dev/requirements_base.txt +++ b/etc/conf/dev/requirements_base.txt @@ -1,13 +1,109 @@ -# Testing -aboutcode-toolkit -apipkg -py -pytest -colorama -execnet -pytest-xdist -bumpversion -coverage -pytest-cov -codecov -pytest-rerunfailures +aboutcode-toolkit==4.0.1 \ + --hash=sha256:ac13e6814854285f501a8bbd361d85a8e133d1b1900a335075b24b0e16c3238b \ + # via -r etc/conf/dev/requirements_base.txt +apipkg==1.5 \ + --hash=sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c \ + # via -r etc/conf/dev/requirements_base.txt, execnet +atomicwrites==1.3.0 \ + --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \ + # via pytest +attrs==19.3.0 \ + --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ + # via pytest +boolean.py==3.7 \ + --hash=sha256:82ae181f9c85cb5c893a5a4daba9f24d60b538a7dd27fd0c6752a77eba4fbeff \ + # via aboutcode-toolkit, license-expression +bumpversion==0.5.4.dev0 \ + --hash=sha256:d70f665b9f8fb4ff3064d1905ff76e31f51dd0aba3fed82ddf2da358e00d7ebc \ + # via -r etc/conf/dev/requirements_base.txt +certifi==2018.4.16 \ + --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \ + # via requests +chardet==3.0.4 \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via requests +click==6.7 \ + --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \ + # via aboutcode-toolkit +codecov==2.0.15 \ + --hash=sha256:ae00d68e18d8a20e9c3288ba3875ae03db3a8e892115bf9b83ef20507732bed4 \ + # via -r etc/conf/dev/requirements_base.txt +colorama==0.3.9 \ + --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \ + # via -r etc/conf/dev/requirements_base.txt +coverage==4.4.1 \ + --hash=sha256:4f99551b229643c9638c028b3a097ffd827f7a6b558356b940c6a3cc847bdb6e \ + --hash=sha256:7a9c44400ee0f3b4546066e0710e1250fd75831adc02ab99dda176ad8726f424 \ + # via -r etc/conf/dev/requirements_base.txt, codecov, pytest-cov +execnet==1.6.0 \ + --hash=sha256:027ee5d961afa01e97b90d6ccc34b4ed976702bc58e7f092b3c513ea288cb6d2 \ + # via -r etc/conf/dev/requirements_base.txt, pytest-xdist +idna==2.6 \ + --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \ + # via requests +importlib-metadata==0.18 \ + --hash=sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7 \ + # via pluggy, pytest +jinja2==2.10 \ + --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \ + # via aboutcode-toolkit +license-expression==1.2 \ + --hash=sha256:6d97906380cecfc758a77f6d38c6760f2afade7e83d2b8295e234fe21f486fb8 \ + # via aboutcode-toolkit +markupsafe==1.0 \ + --hash=sha256:198e2d4a8bfcf9d59edd8e870454c21a7c1fc8e7e4dde85f20f1a033ac539a4e \ + --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \ + # via jinja2 +more-itertools==5.0.0 \ + --hash=sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9 \ + # via pytest +packaging==19.0 \ + --hash=sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3 \ + # via pytest +pluggy==0.12.0 \ + --hash=sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c \ + # via pytest +py==1.8.0 \ + --hash=sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa \ + # via -r etc/conf/dev/requirements_base.txt, pytest +pyparsing==2.4.0 \ + --hash=sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03 \ + # via packaging +pytest-cov==2.7.1 \ + --hash=sha256:2b097cde81a302e1047331b48cadacf23577e431b61e9c6f49a1170bbe3d3da6 \ + # via -r etc/conf/dev/requirements_base.txt +pytest-forked==1.0.2 \ + --hash=sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f \ + # via pytest-xdist +pytest-rerunfailures==7.0 \ + --hash=sha256:1180a0f98975e1e1a2e055c87c1159cbd3bace8ceb71b1e7ffe4ace6121e7801 \ + # via -r etc/conf/dev/requirements_base.txt +pytest-xdist==1.29.0 \ + --hash=sha256:501795cb99e567746f30fe78850533d4cd500c93794128e6ab9988e92a17b1f8 \ + # via -r etc/conf/dev/requirements_base.txt +pytest==4.6.3 \ + --hash=sha256:926855726d8ae8371803f7b2e6ec0a69953d9c6311fa7c3b6c1b929ff92d27da \ + # via -r etc/conf/dev/requirements_base.txt, pytest-cov, pytest-forked, pytest-rerunfailures, pytest-xdist +pyyaml==3.12 \ + --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \ + --hash=sha256:bb35246803f4e91ccb7e160d4c872dc8ea93cd664082033b6feacd72e25653d3 \ + --hash=sha256:d154a918f5da478a1823d0a4190408e4ad450ce8965de8d4913ef37a58e56563 \ + # via aboutcode-toolkit, saneyaml +requests==2.18.4 \ + --hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \ + # via codecov +saneyaml==0.1 \ + --hash=sha256:ffef9eed0de856fc2a1d5807c962d22e2b9bbadc0eba2e0375cb64c145006c96 \ + # via aboutcode-toolkit +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + # via more-itertools, packaging, pytest, pytest-xdist +urllib3==1.22 \ + --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \ + # via requests +wcwidth==0.1.7 \ + --hash=sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c \ + # via pytest +zipp==0.5.1 \ + --hash=sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d \ + # via importlib-metadata diff --git a/etc/conf/requirements-py36_all.txt b/etc/conf/requirements-py36_all.txt new file mode 100644 index 00000000000..c021235e5e6 --- /dev/null +++ b/etc/conf/requirements-py36_all.txt @@ -0,0 +1,253 @@ +attrs==19.3.0 \ + --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \ + # via scancode-toolkit (setup.py) +banal==0.3.3 \ + --hash=sha256:63d979b8b63458dd538f10aca4ea5292b94729f60a08ad6b41b8129a9a9f716a \ + # via normality +beautifulsoup4==4.7.1 \ + --hash=sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858 \ + # via scancode-toolkit (setup.py) +binaryornot==0.4.4 \ + --hash=sha256:b8b71173c917bddcd2c16070412e369c3ed7f0528926f70cac18a6c97fd563e4 \ + # via scancode-toolkit (setup.py) +bitarray==0.8.1 \ + --hash=sha256:1e8612b7244ff0e311e36fd24108935461ebae63fc62fbcb098c2cadb9850892 \ + --hash=sha256:2d33af83a66775d8f6e961ffb63c04a936ddd51d5383c10d34ebcdf737d373e5 \ + --hash=sha256:7da501356e48a83c61f479393681c1bc4b94e5a34ace7e08cb29e7dd9290ab18 \ + --hash=sha256:8db28f3166fa3b0f77b04f9378b5e9a249967cba1d2e487647d1e5ef2353cc27 \ + --hash=sha256:dcae0e9839ca59ec383ba998318584c8cc0b126852e7f217475274a1a8734006 \ + # via scancode-toolkit (setup.py) +boolean.py==3.7 \ + --hash=sha256:82ae181f9c85cb5c893a5a4daba9f24d60b538a7dd27fd0c6752a77eba4fbeff \ + # via license-expression, scancode-toolkit (setup.py) +certifi==2018.4.16 \ + --hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \ + # via requests +chardet==3.0.4 \ + --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \ + # via binaryornot, normality, requests, scancode-toolkit (setup.py) +click==6.7 \ + --hash=sha256:29f99fc6125fbc931b758dc053b3114e55c77a6e4c6c3a2674a2dc986016381d \ + # via scancode-toolkit (setup.py) +colorama==0.3.9 \ + --hash=sha256:463f8483208e921368c9f306094eb6f725c6ca42b0f97e313cb5d5512459feda \ + # via scancode-toolkit (setup.py) +contextlib2==0.5.5 \ + --hash=sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00 \ + # via scancode-toolkit (setup.py) +dparse==0.4.1 \ + --hash=sha256:00a5fdfa900629e5159bf3600d44905b333f4059a3366f28e0dbd13eeab17b19 \ + --hash=sha256:a14ebcee6074d5614245cf6f7dc8d21526412c866d06305745010bd091fd630c \ + # via scancode-toolkit (setup.py) +extractcode-7z==16.2.4.1 \ + --hash=sha256:0c28ce96df713a368af955969efe2b3d5c0581306b5e8b77c6bdee45e23f94f3 \ + --hash=sha256:3eeaa645732505d8bc36cf2d4a38ad3bed03c8da4f14e89217d9495edeecbd5a \ + --hash=sha256:52ab4ba8252f9b79574e730e30d0c12b465b9c796772f01bad77a26958837bdc \ + --hash=sha256:a24094c606fc2a974f84c5293cd7f704c1866383ba0450465b80fc1d4e10f3d7 \ + # via scancode-toolkit (setup.py) +extractcode-libarchive==3.4.2.1 \ + --hash=sha256:558fad3098e5285ac7784a5bbf8463425e2bbb3d4218458b0b2ea53b34e3ff88 \ + --hash=sha256:83fb3f9a101adbf2ea8ce046d13634c2cfd1e7297a33e55feb790cad9844fda5 \ + --hash=sha256:9922f30ce882e21e830ba39f81d9f052b09ee08365807b3f11284036c4a1f1eb \ + --hash=sha256:dc41439b863ccf1f9ed092c870e5537aafa227c23d90bd335382a17e5de9343b \ + # via scancode-toolkit (setup.py) +fingerprints==0.6.5.1 \ + --hash=sha256:a80555655fc249f103369792d8ea1bd5870434b0a10d3a92ade6ad3e908fe4aa \ + # via scancode-toolkit (setup.py) +ftfy==4.4.3 \ + --hash=sha256:061fbc1bf511673cb271ee1022ac17beedd5b169e4f4c3fd5b5e90b5b883dbdc \ + --hash=sha256:3c0066db64a98436e751e56414f03f1cdea54f29364c0632c141c36cca6a5d94 \ + # via scancode-toolkit (setup.py) +future==0.16.0 \ + --hash=sha256:c132ffad1ed08491003ee07fcdc6fab43fa99b02560ad4cdff8e52a141a79bfb \ + # via pefile, scancode-toolkit (setup.py) +gemfileparser==0.8.0 \ + --hash=sha256:839592e49ea3fd985cec003ef58f8e77009a69ed7644a0c0acc94cf6dd9b8d6e \ + --hash=sha256:aed669b729c1f850308d1c373fe425d0c1ce768fa5bdac593e05715c64e2bffc \ + # via scancode-toolkit (setup.py) +html5lib==0.999999999 \ + --hash=sha256:b8934484cf22f1db684c0fae27569a0db404d0208d20163fbf51cc537245d008 \ + # via scancode-toolkit (setup.py) +idna==2.6 \ + --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \ + # via requests +importlib-metadata==0.18 \ + --hash=sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7 \ + # via pluggy +intbitset==2.3.0 \ + --hash=sha256:01505d26ba2b99fe04d2a5b6d166d8ca92f489a02faa6aa825855af047971da8 \ + --hash=sha256:589aff5117f2660767e04499dabb6288ef88e05566d3ac1d9fa3bb7cac4d5ef5 \ + --hash=sha256:5a38b1b338479027b1a75351a39e7d126682a2ea75d210146ffb735d1a767b6f \ + --hash=sha256:7dc2fac52a3888ab29592854511588bdb45ea2247378be78595a59a69afbd5c3 \ + --hash=sha256:db1b85620aa3ad3e2b8d448cf9f088b61c30a4c75f1be46e07eca32fbb407409 \ + # via scancode-toolkit (setup.py) +isodate==0.6.0 \ + --hash=sha256:aa4d33c06640f5352aca96e4b81afd8ab3b47337cc12089822d6f322ac772c81 \ + # via rdflib +jaraco.functools==2.0 \ + --hash=sha256:e9e377644cee5f6f9128b4dab1631fca74981236e95a255f80e4292bcd2b5284 \ + # via scancode-toolkit (setup.py), tempora, yg.lockfile +javaproperties==0.5.1 \ + --hash=sha256:8bfb757116ed0589d88b4e13ecdc7d05a22c647a4645abf43ec27059430b1468 \ + # via scancode-toolkit (setup.py) +jinja2==2.10 \ + --hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \ + # via scancode-toolkit (setup.py) +license-expression==1.2 \ + --hash=sha256:6d97906380cecfc758a77f6d38c6760f2afade7e83d2b8295e234fe21f486fb8 \ + # via scancode-toolkit (setup.py) +lxml==4.2.1 \ + --hash=sha256:0e3cd94c95d30ba9ca3cff40e9b2a14e1a10a4fd8131105b86c6b61648f57e4b \ + --hash=sha256:28f0c6652c1b130f1e576b60532f84b19379485eb8da6185c29bd8c9c9bc97bf \ + --hash=sha256:691f2cd97cf026c611df1ea5055755eec7f878f2d4f4330dc8686583de6fc5fd \ + --hash=sha256:8f37627f16e026523fca326f1b5c9a43534862fede6c3e99c2ba6a776d75c1ab \ + --hash=sha256:e2629cdbcad82b83922a3488937632a4983ecc0fed3e5cfbf430d069382eeb9b \ + # via pymaven-patch, scancode-toolkit (setup.py) +markupsafe==1.0 \ + --hash=sha256:198e2d4a8bfcf9d59edd8e870454c21a7c1fc8e7e4dde85f20f1a033ac539a4e \ + --hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \ + # via jinja2 +more-itertools==5.0.0 \ + --hash=sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9 \ + # via jaraco.functools +nltk==3.2 \ + --hash=sha256:76d756a0c3949796068eb8c29feb7dc7fd534a98a4d348ec39429032c652230f \ + --hash=sha256:cdbf05a212c5ac8ad2fe0eb65a5399ec6979f97206ddfac5e358a0feecec60bc \ + # via scancode-toolkit (setup.py) +normality==0.5.12 \ + --hash=sha256:063fd1b8319241d02d1960f7f1ab112925a10469fcf02231cbe953f021761cf1 \ + # via fingerprints +packageurl-python==0.9.0 \ + --hash=sha256:cf53de6196a1b9d507b077604f9cead63537418634cdd096cc073df1c373ddf7 \ + # via scancode-toolkit (setup.py) +packaging==19.0 \ + --hash=sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3 \ + # via dparse +patch==1.16 \ + --hash=sha256:f132f44ca01557adbd8407d8f1935d9ecc351f12a13f64119254c0358706fc91 \ + # via scancode-toolkit (setup.py) +pdfminer.six==20181108 \ + --hash=sha256:f04d029d1d3e58c87da51bdefef2e9a1dbf2d7b63f727dd2a3e36054f5ae96ea \ + # via scancode-toolkit (setup.py) +pefile==2019.4.18 \ + --hash=sha256:6702305f555558ed0d31c78a03d7c4fc8203b7be63874f7b21b6f779c8bd7b13 \ + --hash=sha256:a5d6e8305c6b210849b47a6174ddf9c452b2888340b8177874b862ba6c207645 \ + # via scancode-toolkit (setup.py) +pkginfo==1.5.0.1 \ + --hash=sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32 \ + # via scancode-toolkit (setup.py) +pluggy==0.12.0 \ + --hash=sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c \ + # via scancode-toolkit (setup.py) +ply==3.9 \ + --hash=sha256:4c76e737669f494cea2040c8a94e89b43949f7c663c2e40686e9d82fca8767f7 \ + # via spdx-tools +publicsuffix2==2.20191221 \ + --hash=sha256:786b5e36205b88758bd3518725ec8cfe7a8173f5269354641f581c6b80a99893 \ + # via scancode-toolkit (setup.py), urlpy +pyahocorasick==1.4.0 \ + --hash=sha256:14f7c371b351196ace8b7cbc3be74363c627a606196fadc3ac0ea917aa58321c \ + --hash=sha256:46aff0f15af1895daa18fc7e7cb77d4ec8041328c4fc66188de730b12605681e \ + --hash=sha256:62114ed7d25bf31e44c232cc0c698575c02a05048ed82afd0ab962a21a5a435d \ + --hash=sha256:90f27fe09eb884fa4fda42d40dccc74a1ad0db9957609f1e0c5b574f315e4e58 \ + # via scancode-toolkit (setup.py) +pycryptodome==3.6.1 \ + --hash=sha256:15013007e393d0cc0e69f4329a47c4c8597b7f3d02c12c03f805405542f70c71 \ + --hash=sha256:4b5a2680008da3ac0cef2d3661597e0cbf8a3eb19eed35b859fd67e2de63eb85 \ + --hash=sha256:e51da4ef9d9e2695a04044152f380c2db17adc9fc6fad8e24d863ead9cd548ed \ + --hash=sha256:e850e07f54dc3de9a1efdd59d227fcd1cb30cdd307dafdc647c79e8f30cf5032 \ + # via pdfminer.six, scancode-toolkit (setup.py) +pygments==2.2.0 \ + --hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \ + # via scancode-toolkit (setup.py) +pymaven-patch==0.2.9 \ + --hash=sha256:478f91bf236f07c6445f03518849936e95f5b1edf52029c323fcf609ec579a65 \ + # via scancode-toolkit (setup.py) +pyparsing==2.4.0 \ + --hash=sha256:9b6323ef4ab914af344ba97510e966d64ba91055d6b9afa6b30799340e89cc03 \ + # via packaging, rdflib +pytz==2019.2 \ + --hash=sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7 \ + # via scancode-toolkit (setup.py), tempora +pyyaml==3.12 \ + --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \ + --hash=sha256:bb35246803f4e91ccb7e160d4c872dc8ea93cd664082033b6feacd72e25653d3 \ + --hash=sha256:d154a918f5da478a1823d0a4190408e4ad450ce8965de8d4913ef37a58e56563 \ + # via dparse, fingerprints, saneyaml, spdx-tools +rdflib==4.2.1 \ + --hash=sha256:a737fd5af85484b74a60aa25dccfb2abaa6032c687645845db360d1f850d8321 \ + # via spdx-tools +requests==2.18.4 \ + --hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \ + # via pymaven-patch, scancode-toolkit (setup.py) +saneyaml==0.1 \ + --hash=sha256:ffef9eed0de856fc2a1d5807c962d22e2b9bbadc0eba2e0375cb64c145006c96 \ + # via scancode-toolkit (setup.py) +simplejson==3.15.0 \ + --hash=sha256:3073b4a18355f6a6260b7bd21f57281d9ed3ff80b6d33ad29f64d5c04863f65f \ + --hash=sha256:7d130c4dd66815ba5483dc620ad72f586e5e12afe6b94e3a7336dbe09a04d809 \ + --hash=sha256:ad332f65d9551ceffc132d0a683f4ffd12e4bc7538681100190d577ced3473fb \ + # via scancode-toolkit (setup.py) +six==1.12.0 \ + --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \ + # via dparse, fingerprints, html5lib, intbitset, javaproperties, more-itertools, normality, packaging, pdfminer.six, pymaven-patch, scancode-toolkit (setup.py), spdx-tools, tempora +sortedcontainers==2.0.5 \ + --hash=sha256:220bb2e3e1886297fd7cdd6d164cb5cf237be1cfae1a3a3e526d149c52816682 \ + # via pdfminer.six +soupsieve==1.9.2 \ + --hash=sha256:e4e9c053d59795e440163733a7fec6c5972210e1790c507e4c7b051d6c5259de \ + # via beautifulsoup4 +spdx-tools==0.6.1 \ + --hash=sha256:45fa09d43534772ca63bff2dccf1fbde88e1fc9b9515fc7a34ae50540d7ad29c \ + # via scancode-toolkit (setup.py) +tempora==1.14.1 \ + --hash=sha256:d28a03d2f64ee81aec6e6bff374127ef306fe00c1b7e27c7ff1618344221a699 \ + # via scancode-toolkit (setup.py), yg.lockfile +text-unidecode==1.2 \ + --hash=sha256:801e38bd550b943563660a91de8d4b6fa5df60a542be9093f7abf819f86050cc \ + # via scancode-toolkit (setup.py) +toml==0.10.0 \ + --hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \ + --hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \ + # via scancode-toolkit (setup.py) +typecode-libmagic==5.38.1 \ + --hash=sha256:434ab7d8dcc9a1de5137dab1e0e3dc2959502815e30c0fd9920f3d3b58ca39dc \ + --hash=sha256:4bfb17fbb8ef035f03ca79d32daa1e05ca13bcc45e56fd7ccda52f496101d6eb \ + --hash=sha256:5bc822547d9537cc67e8bb7472eaf7d02ae172f72f22162e743ec3548974c226 \ + --hash=sha256:f85edf1175ae8d9f26fdf972a66598db0afb88aae4f7f4fa0ef1477091f0f02b \ + # via scancode-toolkit (setup.py) +typing==3.6.4 \ + --hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \ + # via scancode-toolkit (setup.py) +unicodecsv==0.14.1 \ + --hash=sha256:e0e43259d7656da259f264cfa369e66d0643dbd944ffcd001be4df9c21bcf731 \ + # via scancode-toolkit (setup.py) +urllib3==1.22 \ + --hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \ + # via requests +urlpy==0.5 \ + --hash=sha256:841673d97e0dd7a4d7ba47abd49fa8e3a61709e189e40de1b04b150ce7c5ed9f \ + # via scancode-toolkit (setup.py) +wcwidth==0.1.7 \ + --hash=sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c \ + # via ftfy +webencodings==0.5 \ + --hash=sha256:36b29f8752ec757b0a3181a104db8aa327ffd8f1e4996c4c755f97f7837acb05 \ + # via html5lib +xmltodict==0.11.0 \ + --hash=sha256:add07d92089ff611badec526912747cf87afd4f9447af6661aca074eeaf32615 \ + # via scancode-toolkit (setup.py), spdx-tools +yg.lockfile==2.3 \ + --hash=sha256:b3fe554a90a7ee97393bc0d2f27068894c401c91b7d92b76c04348fd13b86570 \ + # via scancode-toolkit (setup.py) +zc.lockfile==2.0 \ + --hash=sha256:cc33599b549f0c8a248cb72f3bf32d77712de1ff7ee8814312eb6456b42c015f \ + # via scancode-toolkit (setup.py), yg.lockfile +zipp==0.5.1 \ + --hash=sha256:8c1019c6aad13642199fbe458275ad6a84907634cc9f0989877ccc4a2840139d \ + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +setuptools==41.2.0 \ + --hash=sha256:4380abcf2a4ffd1a5ba22d687c6d690dce83b2b51c70e9c6d09f7e8c7e8040dc \ + # via html5lib, zc.lockfile diff --git a/etc/conf/requirements-py36_base.txt b/etc/conf/requirements-py36_base.txt new file mode 100644 index 00000000000..65c5ac1c0dd --- /dev/null +++ b/etc/conf/requirements-py36_base.txt @@ -0,0 +1,3 @@ +-r requirements-py36_all.txt +-r dev/requirements_base.txt +#-e . \ No newline at end of file diff --git a/etc/conf/requirements_base.txt b/etc/conf/requirements_base.txt deleted file mode 100644 index ecf975e2fa6..00000000000 --- a/etc/conf/requirements_base.txt +++ /dev/null @@ -1 +0,0 @@ --e . \ No newline at end of file diff --git a/etc/configure.py b/etc/configure.py index 865553becab..7d6a48710ce 100644 --- a/etc/configure.py +++ b/etc/configure.py @@ -12,9 +12,9 @@ To use, create a configuration directory tree that contains any of these: * Requirements files named with this convention: - - requirements_base.txt contains common requirements installed on all platforms. - - requirements_win.txt, requirements_linux.txt, requirements_mac.txt and - requirements_posix.txt are os-specific requirements. + - requirements-py36_base.txt contains common requirements installed on all platforms. + - requirements-py36_win.txt, requirements-py36_linux.txt, requirements-py36_mac.txt and + requirements-py36_posix.txt are os-specific requirements. * Python scripts files named with this convention: - base.py is a common script executed on all os before os-specific scripts. @@ -46,15 +46,15 @@ For example a tree could be looking like this: etc/conf - requirements_base.txt : base pip requirements for all platforms - requirements_linux.txt : linux-only pip requirements + requirements-py36_base.txt : base pip requirements for all platforms + requirements-py36_linux.txt : linux-only pip requirements base.py : base config script for all platforms win.py : windows-only config script posix.sh: posix-only shell script etc/conf/prod requirements_base.txt : base pip requirements for all platforms - requirements_win.txt : Windows-only pip requirements + requirements-py36_win.txt : Windows-only pip requirements linux.sh : linux-only script base.py : base config script for all platforms mac.py : mac-only config script @@ -62,10 +62,10 @@ A call using etc/conf/prod would results in these steps if on linux: 1. Create a virtualenv 2. Run pip install with - etc/conf/requirements_base.txt - etc/conf/requirements_linux.txt + etc/conf/requirements-py36_base.txt + etc/conf/requirements-py36_linux.txt etc/conf/prod/requirements_base.txt - (etc/conf/prod/requirements_win.txt is skipped on linux) + (etc/conf/prod/requirements-py36_win.txt is skipped on linux) 3. Run these Python scripts: etc/conf/base.py (etc/conf/win.py is skipped on linux) @@ -104,10 +104,16 @@ platform_names = tuple() +if sys.version_info < (3, 6): + raise Exception('Only Python 3.6 and above are supported!') + + # Python versions _sys_v0 = sys.version_info[0] +_sys_v1 = sys.version_info[1] py2 = _sys_v0 == 2 py3 = _sys_v0 == 3 +python_version = str(_sys_v0) + str(_sys_v1) # common file basenames for requirements and scripts @@ -115,7 +121,7 @@ # known full file names with txt extension for requirements # base is always last -requirement_filenames = tuple('requirements_' + p + '.txt' for p in platform_names + base) +requirement_filenames = tuple('requirements-' + 'py' + python_version + '_' + p + '.txt' for p in platform_names + base) # known full file names with py extensions for scripts # base is always last @@ -217,10 +223,13 @@ def build_pip_dirs_args(paths, root_dir, option='--extra-search-dir='): list of `paths` to directories. """ for path in paths: - if not os.path.isabs(path): - path = os.path.join(root_dir, path) - if os.path.exists(path): - yield option + quote(path) + if path.startswith('https'): + yield option + '"{}"'.format(path) + else: + if not os.path.isabs(path): + path = os.path.join(root_dir, path) + if os.path.exists(path): + yield option + quote(path) def create_virtualenv(std_python, root_dir, tpp_dirs=(), quiet=False): @@ -246,6 +255,8 @@ def create_virtualenv(std_python, root_dir, tpp_dirs=(), quiet=False): # search the virtualenv.pyz app in the tpp_dirs. keep the first found venv_pyz = None for tpd in tpp_dirs: + if tpd.startswith('https'): + continue venv = os.path.join(root_dir, tpd, 'virtualenv.pyz') if os.path.exists(venv): venv_pyz = venv @@ -253,14 +264,12 @@ def create_virtualenv(std_python, root_dir, tpp_dirs=(), quiet=False): # error out if venv_pyz not found if not venv_pyz: - print("Configuration Error ... aborting.") + print("Configuration Error: Unable to find virtualenv.pyz ... aborting.") exit(1) vcmd = [quote(std_python), quote(venv_pyz), '--never-download'] if quiet: vcmd += ['-qq'] - # third parties may be in more than one directory - vcmd.extend(build_pip_dirs_args(tpp_dirs, root_dir)) # we create the virtualenv in the root_dir vcmd.append(quote(root_dir)) call(vcmd, root_dir) @@ -286,6 +295,15 @@ def install_3pp(configs, root_dir, tpp_dirs, quiet=False): run_pip(requirements, root_dir, tpp_dirs, quiet) +def install_local_package(root_dir, tpp_dirs, quiet=False): + """ + Install the current local package with pip, + using the vendored components in `tpp_dirs`. + """ + requirements = ['--editable', '.'] + run_pip(requirements, root_dir, tpp_dirs, quiet) + + def run_pip(requirements, root_dir, tpp_dirs, quiet=False): """ Install a list of `requirements` with pip, @@ -540,26 +558,30 @@ def save_activate_this_py_script(activate_path): for envvar, path in os.environ.items(): if not envvar.startswith('TPP_DIR'): continue - abs_path = path - if not os.path.isabs(path): - abs_path = os.path.join(root_dir, path) - if not os.path.exists(abs_path): - if not quiet: - print() - print( - 'WARNING: Third-party Python libraries directory does not exists:\n' - ' %(path)r: %(abs_path)r\n' - ' Provided by environment variable:\n' - ' set %(envvar)s=%(path)r' % locals()) - print() - else: + if path.startswith('https'): thirdparty_dirs.append(path) + else: + abs_path = path + if not os.path.isabs(path): + abs_path = os.path.join(root_dir, path) + if not os.path.exists(abs_path): + if not quiet: + print() + print( + 'WARNING: Third-party Python libraries directory does not exists:\n' + ' %(path)r: %(abs_path)r\n' + ' Provided by environment variable:\n' + ' set %(envvar)s=%(path)r' % locals()) + print() + else: + thirdparty_dirs.append(path) # Finally execute our three steps: venv, install and scripts if not os.path.exists(configured_python): create_virtualenv(standard_python, root_dir, thirdparty_dirs, quiet=quiet) activate(root_dir) install_3pp(configs, root_dir, thirdparty_dirs, quiet=quiet) + install_local_package(root_dir, thirdparty_dirs, quiet=quiet) run_scripts(configs, root_dir, configured_python, quiet=quiet) if not quiet: diff --git a/etc/release/MANIFEST.in.release b/etc/release/MANIFEST.in.release index 9c4103685e0..03b407472f9 100644 --- a/etc/release/MANIFEST.in.release +++ b/etc/release/MANIFEST.in.release @@ -3,8 +3,8 @@ graft src graft thirdparty graft plugins graft samples -graft docs +prune docs prune thirdparty/dev prune tests prune plugins-builtin diff --git a/etc/release/release.sh b/etc/release/release.sh index f99bb31ce76..ce414c5e0c0 100755 --- a/etc/release/release.sh +++ b/etc/release/release.sh @@ -13,27 +13,72 @@ set -x echo "### BUILDING ScanCode release ###" +PYPI_REPO=https://github.com/Abhishek-Dev09/thirdparty/releases/tag/v2.0 + echo " RELEASE: Cleaning previous release archives, then setup and config: " rm -rf dist/ build/ # backup dev manifests cp MANIFEST.in MANIFEST.in.dev +# backup thirdparty +cp -r thirdparty thirdparty_dev +rm -rf thirdparty + # install release manifests cp etc/release/MANIFEST.in.release MANIFEST.in +python_version=`python -c "import sys;t='py{v[0]}{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)";` + +# download all dependencies as per OS/arch/python +python3 etc/scripts/deps_download.py --find-links $PYPI_REPO --requirement etc/conf/requirements-"$python_version"_all.txt --dest thirdparty + +if [ "$(uname -s)" == "Darwin" ]; then + platform="mac" +elif [ "$(uname -s)" == "Linux" ]; then + platform="linux" +# FIXME: This may only works on azure not elsewhere +elif [ "$(uname -s)" == "MINGW32_NT" ]; then + platform="win32" +elif [ "$(uname -s)" == "MINGW64_NT" ]; then + platform="win64" +fi + +#copy virtual env files +cp thirdparty_dev/{virtualenv.pyz,virtualenv.pyz.ABOUT} thirdparty + ./configure --clean export CONFIGURE_QUIET=1 ./configure etc/conf +# create requirements files as per OS/arch/python +source bin/activate +pip install -r etc/scripts/req_tools.txt +python etc/scripts/freeze_and_update_reqs.py --find-links thirdparty --requirement etc/conf/requirements-"$python_version"_"$platform".txt + echo " RELEASE: Building release archives..." # build a zip and tar.bz2 bin/python setup.py --quiet --use-default-version clean --all sdist --formats=bztar,zip bdist_wheel +# rename release archive as per os/arch/python +# See https://unix.stackexchange.com/questions/121570/rename-multiples-files-using-bash-scripting +for f in dist/scancode-toolkit*.tar.bz2 + do + mv "$f" "${f%*.*.*}-"$python_version"_"$platform".tar.bz2" + done + +for f in dist/scancode-toolkit*.zip + do + mv "$f" "${f%*.*}-"$python_version"_"$platform".zip" + done + # restore dev manifests mv MANIFEST.in.dev MANIFEST.in +# restore thirdparty +rm -rf thirdparty +mv thirdparty_dev thirdparty function test_scan { # run a test scan for a given archive diff --git a/etc/scripts/deps_download.py b/etc/scripts/deps_download.py new file mode 100644 index 00000000000..2846b8c0402 --- /dev/null +++ b/etc/scripts/deps_download.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/scancode-toolkit/ +# The ScanCode software is licensed under the Apache License version 2.0. +# Data generated with ScanCode require an acknowledgment. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with ScanCode or any ScanCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# ScanCode is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode-toolkit/ for support and download. + +from __future__ import absolute_import +from __future__ import print_function + +import argparse +import os +from subprocess import run +import sys + +def download_req_tools(links, requirement, dir_name): + """ + Download all dependencies into dir_name as per + OS/arch/python by taking links,requirement as an input. + """ + pip_args =[ + 'pip', + 'download', + '--no-cache-dir', + '--no-index', + '--find-links', + links, + '-r', + requirement, + '--dest', + dir_name, + ] + run(pip_args) + + +def main_with_args(args: str) -> None: + parser = argparse.ArgumentParser( + description="""Download all dependencies into directory + (for instance thirdparty directory) as per OS/arch/python by + taking links,requirement as an input.""", + ) + + parser.add_argument( + '--find-links', + help="A directory or URL where to find packages. See pip help for details.", + type=str, + required=True, + ) + + parser.add_argument( + '--requirement', + help='An existing requirement file (with hashes) listing required packages.', + type=str, + required=True, + ) + + parser.add_argument( + '--dest', + help='Download packages into given directory(Destination)', + type=str, + required=True, + ) + + args = parser.parse_args() + + find_links = args.find_links + requirement = args.requirement + dest = args.dest + download_req_tools(find_links, requirement, dest) + + +def main() -> None: + main_with_args(sys.argv[1:]) + + +if __name__ == '__main__': + main() diff --git a/etc/scripts/freeze_and_update_reqs.py b/etc/scripts/freeze_and_update_reqs.py index 8062932e5dd..d57301a9fc2 100644 --- a/etc/scripts/freeze_and_update_reqs.py +++ b/etc/scripts/freeze_and_update_reqs.py @@ -69,6 +69,8 @@ def generate_req_text(find_links, req_file, package_name=None, upgrade=False): '--output-file', req_file, '--allow-unsafe', + '--no-emit-find-links', + '--no-header', '--pip-args', '--no-index', ] diff --git a/setup.py b/setup.py index aa3b652381a..26ade2758ee 100644 --- a/setup.py +++ b/setup.py @@ -137,7 +137,7 @@ def read(*names, **kwargs): # cluecode # Some nltk version ranges are buggy 'nltk >= 3.2, < 4.0', - 'py2_ipaddress >= 2.0, <3.5; python_version<"3"', + 'py2_ipaddress >= 2.0, <3.5; python_version < "3"', 'urlpy', 'publicsuffix2', 'fingerprints >= 0.6.0, < 1.0.0', @@ -146,12 +146,12 @@ def read(*names, **kwargs): 'patch >= 1.15, < 1.20 ', # to work around bug http://bugs.python.org/issue19839 # on multistream bzip2 files: this can removed in Python 3. - 'bz2file >= 0.98; python_version<"3"', + 'bz2file >= 0.98; python_version < "3"', 'extractcode_libarchive', 'extractcode_7z', # commoncode - 'backports.os == 0.1.1; python_version<"3"', + 'backports.os == 0.1.1; python_version < "3"', 'future >= 0.16.0', 'text_unidecode >= 1.0, < 2.0', 'saneyaml', @@ -192,8 +192,8 @@ def read(*names, **kwargs): 'dparse >= 0.4.1', # used to fix mojibake in Windows PE - 'ftfy < 5.0.0; python_version == "2.7"', - 'ftfy>= 5.0.0; python_version > "3"', + # for now we use the evrsion that works on both Python 2 and 3 + 'ftfy < 5.0.0', # scancode 'click >= 6.0.0, < 7.0.0', @@ -210,8 +210,8 @@ def read(*names, **kwargs): # ScanCode caching and locking 'yg.lockfile >= 2.3, < 3.0.0', - # used by yg.lockfile - 'contextlib2', 'pytz', 'tempora', 'jaraco.functools', + # used by yg.lockfile + 'contextlib2', 'pytz', 'tempora', 'jaraco.functools', 'zc.lockfile >= 2.0.0, < 3.0.0', ], diff --git a/src/extractcode/uncompress.py b/src/extractcode/uncompress.py index 4b23f7a588a..a43fc6635f0 100644 --- a/src/extractcode/uncompress.py +++ b/src/extractcode/uncompress.py @@ -30,6 +30,7 @@ import gzip import logging import os +from shutil import move try: # These imports add support for multistream BZ2 files @@ -68,7 +69,7 @@ def uncompress(location, target_dir, decompressor, suffix=EXTRACT_SUFFIX): target_location = os.path.join(target_dir, os.path.basename(location) + suffix) if os.path.exists(target_location): fileutils.delete(target_location) - os.rename(tmp_loc, target_location) + move(tmp_loc, target_location) return warnings diff --git a/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT b/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT new file mode 100644 index 00000000000..ed68add3b87 --- /dev/null +++ b/thirdparty/ftfy-4.4.3-py2-none-any.whl.ABOUT @@ -0,0 +1,12 @@ +name: ftfy +version: 4.4.3 +about_resource: ftfy-4.4.3-py2-none-any.whl +download_url: https://files.pythonhosted.org/packages/21/5d/9385540977b00df1f3a0c0f07b7e6c15b5e7a3109d7f6ae78a0a764dab22/ftfy-4.4.3.tar.gz +license_expression: mit +copyright: Copyright (c) Rob Speer (rob@luminoso.com) +attribute: yes +package_url: pkg:pypi/ftfy@4.4.3 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/thirdparty/ftfy-4.4.3-py3-none-any.whl b/thirdparty/ftfy-4.4.3-py3-none-any.whl new file mode 100644 index 00000000000..8d171e65f73 Binary files /dev/null and b/thirdparty/ftfy-4.4.3-py3-none-any.whl differ diff --git a/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT b/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT new file mode 100644 index 00000000000..ae2a1d82109 --- /dev/null +++ b/thirdparty/ftfy-4.4.3-py3-none-any.whl.ABOUT @@ -0,0 +1,12 @@ +name: ftfy +version: 4.4.3 +about_resource: ftfy-4.4.3-py3-none-any.whl +download_url: https://files.pythonhosted.org/packages/21/5d/9385540977b00df1f3a0c0f07b7e6c15b5e7a3109d7f6ae78a0a764dab22/ftfy-4.4.3.tar.gz +license_expression: mit +copyright: Copyright (c) Rob Speer (rob@luminoso.com) +attribute: yes +package_url: pkg:pypi/ftfy@4.4.3 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE diff --git a/thirdparty/ftfy-4.4.3.tar.gz b/thirdparty/ftfy-4.4.3.tar.gz new file mode 100644 index 00000000000..63df7176fc7 Binary files /dev/null and b/thirdparty/ftfy-4.4.3.tar.gz differ diff --git a/thirdparty/ftfy-5.5.1-py3-none-any.whl b/thirdparty/ftfy-5.5.1-py3-none-any.whl deleted file mode 100644 index c15b09cc9e4..00000000000 Binary files a/thirdparty/ftfy-5.5.1-py3-none-any.whl and /dev/null differ diff --git a/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT b/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT deleted file mode 100644 index 8d89b047fce..00000000000 --- a/thirdparty/ftfy-5.5.1-py3-none-any.whl.ABOUT +++ /dev/null @@ -1,14 +0,0 @@ -name: ftfy -version: 5.5.1 -about_resource: ftfy-5.5.1-py3-none-any.whl -download_url: https://files.pythonhosted.org/packages/8f/86/df789c5834f15ae1ca53a8d4c1fc4788676c2e32112f6a786f2625d9c6e6/ftfy-5.5.1-py3-none-any.whl -license_expression: mit -copyright: Copyright (c) Robyn Speer (rspeer@luminoso.com) -attribute: yes -checksum_md5: 9b9716f707a0984f97f61569831da556 -checksum_sha1: 4b7a7c9bb0f5f3f2d9e1dd8564ea1e2681df815e -package_url: pkg:pypi/ftfy@5.5.1 -licenses: - - key: mit - name: MIT License - file: mit.LICENSE