Merge branch 'develop' into license-detection-improvements-and-review

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

.github/workflows/about-files-ci.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/docs-ci.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 

.github/workflows/scancode-release.yml

@@ -37,10 +37,10 @@ jobs:
        pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python on ${{ matrix.pyver }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }} 
 
@@ -69,10 +69,10 @@ jobs:
       fail-fast: true
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
@@ -103,10 +103,10 @@ jobs:
        pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python on ${{ matrix.pyver }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }} 
 
@@ -137,10 +137,10 @@ jobs:
        pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python on ${{ matrix.pyver }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }} 
 
@@ -171,10 +171,10 @@ jobs:
        pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python on ${{ matrix.pyver }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }} 
 
@@ -202,10 +202,10 @@ jobs:
       fail-fast: true
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: "3.10"
 
@@ -239,10 +239,10 @@ jobs:
         pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.pyver }} on ${{ matrix.os }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }}
 
@@ -285,10 +285,10 @@ jobs:
        pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.pyver }} on ${{ matrix.os }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }}
 
@@ -330,10 +330,10 @@ jobs:
         pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.pyver }} on ${{ matrix.os }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }}
 
@@ -371,10 +371,10 @@ jobs:
         pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.pyver }} on ${{ matrix.os }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }}
 
@@ -412,10 +412,10 @@ jobs:
         pyver: ["3.7", "3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Set up Python ${{ matrix.pyver }} on ${{ matrix.os }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.pyver }}
 
@@ -577,7 +577,7 @@ jobs:
 
     steps:
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: 3.8
 

.readthedocs.yml

@@ -5,6 +5,11 @@
 # Required
 version: 2
 
+# Build PDF & ePub
+formats:
+  - epub
+  - pdf
+
 # Where the Sphinx conf.py file is located
 sphinx:
    configuration: docs/source/conf.py

CHANGELOG.rst

@@ -42,6 +42,28 @@ v32.1.0 (next, roadmap)
   See https://github.com/nexB/scancode-toolkit/issues/1745
 
 
+v32.0.4 - 2023-06-07
+---------------------
+
+This is a minor bugfix release with the following updates:
+
+- Fixes a performance issue issue arising out of license detection
+  on files happening in a single-threaded process_codebase step when the
+  license CLI option is disabled for a package scan.
+  Reference: https://github.com/nexB/scancode-toolkit/pull/3423
+
+v32.0.3 - 2023-06-06
+---------------------
+
+This is a minor bugfix release with the following updates:
+
+- We were missing scancode-toolkit-mini releases from v32.0.0rc2 and
+  also the scancode-toolkit release wheels including and after v32.0.0rc2 were
+  actually scancode-toolkit-mini releases.
+  Reference: https://github.com/nexB/scancode-toolkit/issues/3421
+
+- Updated github actions, for more details see https://github.com/nexB/skeleton/issues/75
+
 v32.0.2 - 2023-05-26
 ---------------------
 

docs/source/conf.py

@@ -104,3 +104,9 @@
 .. role:: img-title-para
 
 """
+
+# -- Options for LaTeX output -------------------------------------------------
+
+latex_elements = {
+    'classoptions': ',openany,oneside'
+}

setup-mini.cfg

@@ -1,6 +1,6 @@
 [metadata]
-name = scancode-toolkit
-version = 32.0.2
+name = scancode-toolkit-mini
+version = 32.0.4
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = scancode-toolkit
-version = 32.0.2
+version = 32.0.4
 license = Apache-2.0 AND CC-BY-4.0 AND LicenseRef-scancode-other-permissive AND LicenseRef-scancode-other-copyleft
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

src/packagedcode/build.py

@@ -14,6 +14,8 @@
 
 from commoncode import fileutils
 
+from licensedcode.cache import build_spdx_license_expression
+from licensedcode.cache import get_cache
 from licensedcode.tokenize import query_tokenizer
 from licensedcode.detection import detect_licenses
 from licensedcode.detection import get_unknown_license_detection
@@ -115,13 +117,17 @@ def assemble(cls, package_data, resource, codebase, package_adder):
             if TRACE:
                 logger_debug(f"BaseStarlarkManifestHandler.assemble: package_data: {package_data.to_dict()}")
 
-            detections, expression = get_license_detections_and_expression(
-                package=package_data,
-                resource=resource,
-                codebase=codebase,
-            )
-            package.license_detections = detections
-            package.declared_license_expression = expression
+            package.license_detections, package.declared_license_expression = \
+                get_license_detections_and_expression(
+                    package=package_data,
+                    resource=resource,
+                    codebase=codebase,
+                )
+            if package.declared_license_expression:
+                package.declared_license_expression_spdx = str(build_spdx_license_expression(
+                    license_expression=package.declared_license_expression,
+                    licensing=get_cache().licensing,
+                ))
 
             cls.assign_package_to_resources(
                 package=package,
@@ -132,6 +138,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
             yield package
 
+
         # we yield this as we do not want this further processed
         yield resource
 

src/packagedcode/licensing.py

@@ -56,7 +56,7 @@ def logger_debug(*args):
         return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
 
 
-def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
+def add_referenced_license_matches_for_package(resource, codebase):
     """
     Return an updated ``resource`` saving it in place, after adding new license
     detections to the package manifests detected in this resource, following their
@@ -106,13 +106,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
                 if not referenced_resource:
                     continue
 
-                if no_licenses:
-                    referenced_license_detections = get_license_detection_mappings(
-                        location=referenced_resource.location
-                    )
-
-                else:
-                    referenced_license_detections = referenced_resource.license_detections
+                referenced_license_detections = referenced_resource.license_detections
 
                 if referenced_license_detections:
                     modified = True
@@ -160,7 +154,7 @@ def add_referenced_license_matches_for_package(resource, codebase, no_licenses):
             yield resource
 
 
-def add_referenced_license_detection_from_package(resource, codebase, no_licenses):
+def add_referenced_license_detection_from_package(resource, codebase):
     """
     Return an updated ``resource`` saving it in place, after adding new license
     matches (licenses and license_expressions) following their Rule
@@ -209,7 +203,6 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
             sibling_license_detections, _le = get_license_detections_from_sibling_file(
                 resource=root_resource,
                 codebase=codebase,
-                no_licenses=no_licenses,
             )
             if TRACE:
                 logger_debug(
@@ -278,12 +271,10 @@ def add_referenced_license_detection_from_package(resource, codebase, no_license
         yield resource
 
 
-def add_license_from_sibling_file(resource, codebase, no_licenses):
+def add_license_from_sibling_file(resource, codebase):
     """
     Given a resource and it's codebase object, assign licenses to the package
     detections in that resource, from the sibling files of it.
-
-    If `no_license` is True, then license scan (for resources) is disabled.
     """
     if TRACE:
         logger_debug(f'packagedcode.licensing: add_license_from_sibling_file: resource: {resource.path}')
@@ -303,7 +294,6 @@ def add_license_from_sibling_file(resource, codebase, no_licenses):
     license_detections, license_expression = get_license_detections_from_sibling_file(
         resource=resource,
         codebase=codebase,
-        no_licenses=no_licenses,
     )
     if not license_detections:
         return
@@ -333,13 +323,11 @@ def is_legal_or_readme(resource):
     return False
 
 
-def get_license_detections_from_sibling_file(resource, codebase, no_licenses):
+def get_license_detections_from_sibling_file(resource, codebase):
     """
     Return `license_detections`, a list of LicenseDetection objects and a
     `license_expression`, given a resource and it's codebase object, from
     the sibling files of the resource.
-
-    If `no_license` is True, then license scan (for resources) is disabled.
     """
     siblings = []
 
@@ -357,15 +345,7 @@ def get_license_detections_from_sibling_file(resource, codebase, no_licenses):
 
     license_detections = []
     for sibling in siblings:
-        if no_licenses:
-            detections = get_license_detection_mappings(
-                location=sibling.location,
-                analysis=DetectionCategory.PACKAGE_ADD_FROM_SIBLING_FILE.value,
-                post_scan=True,
-            )
-            license_detections.extend(detections)
-        else:
-            license_detections.extend(sibling.license_detections)
+        license_detections.extend(sibling.license_detections)
 
     if not license_detections:
         return [], None