Prepare release

CHANGELOG.rst

@@ -1,6 +1,17 @@
 Release notes
 =============
 
+Version 32.0.0 - (2024-09-05)
+-----------------------------
+
+- Add new optional argument to generate YAML test data files from a template
+- Migrate URLs to new aboutcode.org org
+- Drop support for Python 3.7
+- Replace charset_normalizer by chardet because of unstable behavior between minor versions
+  See https://github.com/jawah/charset_normalizer/issues/520
+- Adopt black and isort style
+
+
 Version 31.2.1 - (2024-05-16)
 -----------------------------
 

azure-pipelines.yml

@@ -13,47 +13,47 @@ jobs:
     parameters:
       job_name: ubuntu20_cpython
       image_name: ubuntu-20.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: ubuntu22_cpython
       image_name: ubuntu-22.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: macos12_cpython
       image_name: macOS-12
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-posix.yml
     parameters:
       job_name: macos13_cpython
       image_name: macOS-13
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2019_cpython
       image_name: windows-2019
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2022_cpython
       image_name: windows-2022
-      python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         all: venv\Scripts\pytest -n 2 -vvs
 
@@ -65,7 +65,7 @@ jobs:
     parameters:
       job_name: ubuntu20_test_all_supported_click_versions
       image_name: ubuntu-20.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
         click_versions: |
           for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7;
@@ -82,18 +82,20 @@ jobs:
     parameters:
       job_name: ubuntu20_cpython_latest_from_pip
       image_name: ubuntu-20.04
-      python_versions: ["3.7", "3.8", "3.9", "3.10"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
-        all:
-          venv/bin/pip install --upgrade-strategy eager --force-reinstall
-          --upgrade -e . && venv/bin/pytest -n 2 -vvs
+        all: |
+          venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .
+          venv/bin/pip freeze
+          venv/bin/pytest -n 2 -vvs
 
   - template: etc/ci/azure-win.yml
     parameters:
       job_name: win2019_cpython_latest_from_pip
       image_name: windows-2019
-      python_versions: ["3.7", "3.8", "3.9", "3.10"]
+      python_versions: ["3.8", "3.9", "3.10", "3.11"]
       test_suites:
-        all:
-          venv\Scripts\pip install --upgrade-strategy eager --force-reinstall
-          --upgrade -e . &&  venv\Scripts\pytest -n 2 -vvs
+        all: |
+          venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .
+          venv\Scripts\pip freeze
+          venv\Scripts\pytest -n 2 -vvs

requirements.txt

@@ -1,7 +1,7 @@
 attrs==22.2.0
 beautifulsoup4==4.11.2
 certifi==2022.12.7
-charset-normalizer==3.0.1
+chardet==5.2.0
 click==8.1.3
 idna==3.4
 pip==23.0

setup.cfg

@@ -37,13 +37,13 @@ zip_safe = false
 
 setup_requires = setuptools_scm[toml] >= 4
 
-python_requires = >=3.7
+python_requires = >=3.8
 
 install_requires =
     attrs >= 18.1, !=20.1.0
-    Beautifulsoup4 >= 4.0.0
+    Beautifulsoup4[chardet] >= 4.0.0
     click >= 6.7, !=7.0
-    requests >= 2.7.0
+    requests[use_chardet_on_py3] >= 2.7.0
     saneyaml >= 0.5.2
     text_unidecode >= 1.0
 

src/commoncode/__init__.py

@@ -14,16 +14,16 @@ def set_re_max_cache(max_cache=1000000):
     libraries use a lot of regexes: therefore 100 is not enough to benefit from
     caching.
     """
-    import re
     import fnmatch
+    import re
 
-    remax = getattr(re, '_MAXCACHE', 0)
+    remax = getattr(re, "_MAXCACHE", 0)
     if remax < max_cache:
-        setattr(re, '_MAXCACHE', max_cache)
+        setattr(re, "_MAXCACHE", max_cache)
 
-    fnmatchmax = getattr(fnmatch, '_MAXCACHE', 0)
+    fnmatchmax = getattr(fnmatch, "_MAXCACHE", 0)
     if fnmatchmax < max_cache:
-        setattr(fnmatch, '_MAXCACHE', max_cache)
+        setattr(fnmatch, "_MAXCACHE", max_cache)
 
 
 set_re_max_cache()

src/commoncode/archive.py

@@ -6,12 +6,12 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from functools import partial
-import os
-from os import path
 import gzip
+import os
 import tarfile
 import zipfile
+from functools import partial
+from os import path
 
 from commoncode.system import on_windows
 
@@ -47,7 +47,7 @@ def extract_tar(location, target_dir, verbatim=False, *args, **kwargs):
     # always for using bytes for paths on all OSses... tar seems to use bytes internally
     # and get confused otherwise
     location = os.fsencode(location)
-    with open(location, 'rb') as input_tar:
+    with open(location, "rb") as input_tar:
         tar = None
         try:
             tar = tarfile.open(fileobj=input_tar)
@@ -69,7 +69,7 @@ def extract_zip(location, target_dir, *args, **kwargs):
     Extract a zip archive file at location in the target_dir directory.
     """
     if not path.isfile(location) and zipfile.is_zipfile(location):
-        raise Exception('Incorrect zip file %(location)r' % locals())
+        raise Exception("Incorrect zip file %(location)r" % locals())
 
     with zipfile.ZipFile(location) as zipf:
         for info in zipf.infolist():
@@ -82,7 +82,7 @@ def extract_zip(location, target_dir, *args, **kwargs):
                 if not path.exists(target):
                     os.makedirs(target)
             if not path.exists(target):
-                with open(target, 'wb') as f:
+                with open(target, "wb") as f:
                     f.write(content)
 
 
@@ -92,7 +92,7 @@ def extract_zip_raw(location, target_dir, *args, **kwargs):
     Use the builtin extractall function
     """
     if not path.isfile(location) and zipfile.is_zipfile(location):
-        raise Exception('Incorrect zip file %(location)r' % locals())
+        raise Exception("Incorrect zip file %(location)r" % locals())
 
     with zipfile.ZipFile(location) as zipf:
         zipf.extractall(path=target_dir)
@@ -124,6 +124,6 @@ def get_gz_compressed_file_content(location):
     Uncompress a compressed file at `location` and return its content as a byte
     string. Raise Exceptions on errors.
     """
-    with gzip.GzipFile(location, 'rb') as compressed:
+    with gzip.GzipFile(location, "rb") as compressed:
         content = compressed.read()
     return content