Merge remote-tracking branch 'origin/main' into 667_mine_nuget_pipeline

Author: keshav-space

minecode/collectors/dockerhub.py

@@ -0,0 +1,147 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import requests
+from packageurl import PackageURL
+
+from minecode import priority_router
+from minecode.miners.dockerhub import build_package_data
+from packagedb.models import PackageContentType
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def fetch_dockerhub_repo_summary(name, namespace="library"):
+    """
+    Fetch summary metadata for a Docker Hub repository.
+
+    Returns:
+        dict or None: Full metadata JSON from the Docker Hub API, including:
+            - description (str): Short description
+            - full_description (str): Detailed description
+            - is_private (bool): Privacy status
+
+    """
+    url = f"https://hub.docker.com/v2/repositories/{namespace}/{name}/"
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as err:
+        logger.error(f"Error fetching repository metadata for {name}: {err}")
+        return None
+
+
+def fetch_dockerhub_tags_metadata(name, namespace, tag=None):
+    """
+    Search through Docker Hub tags for a given repository.
+    - If `tag` is provided, return the JSON metadata for that tag (by name or digest).
+    - If `tag` is None, return a list of all tag metadata.
+
+    Examples:
+        fetch_dockerhub_tag_metadata("nginx", "1.25.2")
+        fetch_dockerhub_tag_metadata("nginx", "sha256:3d8957cb61d0223de2ab1aa2ec91d29796eb82a81cdcc1e968c090c29606d648")
+        fetch_dockerhub_tag_metadata("nginx")  # returns all tags
+
+    """
+    page = 0
+    page_size = 100
+    all_results = []
+
+    while True:
+        page += 1
+        url = f"https://hub.docker.com/v2/repositories/{namespace}/{name}/tags/?page={page}&page_size={page_size}"
+        try:
+            response = requests.get(url)
+            response.raise_for_status()
+            data = response.json()
+
+            results = data.get("results", [])
+            if not tag:
+                all_results.extend(results)  # collect everything
+            else:
+                for result in results:
+                    if tag.startswith("sha256") and result.get("digest") == tag:
+                        return [result]
+                    elif result.get("name") == tag:
+                        return [result]
+
+            # Check if more pages exist
+            if not data.get("next") or page_size * page > data.get("count", 0):
+                break  # no more pages
+
+        except requests.exceptions.RequestException as err:
+            logger.error(f"Error fetching tags for {name}, page {page}: {err}")
+            return None
+
+    if not tag:
+        return all_results  # return collected list
+
+    return None  # tag not found
+
+
+def map_dockerhub_package(package_url, pipelines, priority=0):
+    """
+    Add a Dockerhub distribution `package_url` to the PackageDB.
+    """
+    from minecode.model_utils import add_package_to_scan_queue
+    from minecode.model_utils import merge_or_create_package
+
+    if not package_url.name:
+        error = f"Missing package name in DockerHub Package URL: {package_url}"
+        logger.error(error)
+        return error
+
+    namespace = package_url.namespace or "library"
+    summary = fetch_dockerhub_repo_summary(package_url.name, namespace)
+    if not summary:
+        error = f"Package does not exist on dockerhub: {package_url}"
+        logger.error(error)
+        return error
+
+    tags_metadata = fetch_dockerhub_tags_metadata(package_url.name, namespace, package_url.version)
+
+    packages = build_package_data(summary, tags_metadata, package_url)
+
+    error = None
+    for package in packages:
+        package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+        db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+        if error:
+            break
+
+        if db_package:
+            add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
+    return error
+
+
+@priority_router.route("pkg:docker/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process Dockerhub Package URL (PURL).
+    ex:
+    pkg:docker/nginx@latest
+    pkg:docker/nginx@sha256:3d8957cb61d0223de2ab1aa2ec91d29796eb82a81cdcc1e968c090c29606d648
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+
+    error_msg = map_dockerhub_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/collectors/hex.py

@@ -0,0 +1,84 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+import requests
+from packageurl import PackageURL
+
+from minecode.miners.hex import build_packages
+from minecode import priority_router
+from packagedb.models import PackageContentType
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def get_hex_package_json(name):
+    """
+    Return the metadata JSON for a package from hex.pm API.
+    Example: https://hex.pm/api/packages/phoenix
+    """
+
+    url = f"https://hex.pm/api/packages/{name}"
+
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.HTTPError as err:
+        logger.error(f"HTTP error occurred: {err}")
+
+
+def map_hex_package(package_url, pipelines, priority=0):
+    """
+    Add a hex `package_url` to the PackageDB.
+    """
+    from minecode.model_utils import add_package_to_scan_queue, merge_or_create_package
+
+    name = package_url.name
+    package_json = get_hex_package_json(name=name)
+
+    if not package_json:
+        error = f"Package does not exist on hex.pm: {package_url}"
+        logger.error(error)
+        return error
+
+    packages = build_packages(metadata_dict=package_json, purl=package_url)
+
+    error = None
+    for package in packages:
+        package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+        db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+        if error:
+            break
+        if db_package:
+            add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
+
+    return error
+
+
+@priority_router.route("pkg:hex/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process `priority_resource_uri` containing a hex Package URL (PURL).
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+
+    error_msg = map_hex_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/collectors/pub.py

@@ -60,7 +60,6 @@ def map_pub_package(package_url, pipelines, priority=0):
         db_package, _, _, error = merge_or_create_package(package, visit_level=0)
         if error:
             break
-        print(db_package)
         if db_package:
             add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
 

minecode/collectors/swift.py

@@ -0,0 +1,69 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0
+# See https://github.com/aboutcode-org/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from packageurl import PackageURL
+from minecode import priority_router
+from minecode.miners import github
+from minecode.miners.github import build_github_packages
+from packagedb.models import PackageContentType
+
+logger = logging.getLogger(__name__)
+handler = logging.StreamHandler()
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def map_swift_package(package_url, pipelines, priority=0):
+    """
+    Add a Swift distribution `package_url` to the PackageDB.
+    """
+    from minecode.model_utils import add_package_to_scan_queue, merge_or_create_package
+
+    namespace = package_url.namespace
+    version = package_url.version
+
+    owner_name = namespace.split("/")[-1]
+
+    uri = f"https://api.github.com/repos/{owner_name}/{package_url.name}"
+    _, response_text, _ = github.GithubSingleRepoVisitor(uri)
+    repo_data = json.loads(response_text)
+    repo_data["tags"] = [tag for tag in repo_data["tags"] if tag["name"] == version]
+    packages = build_github_packages(json.dumps(repo_data), uri, package_url)
+
+    error = None
+    for package in packages:
+        package.type = "swift"
+        package.namespace = namespace
+        package.extra_data["package_content"] = PackageContentType.SOURCE_ARCHIVE
+        db_package, _, _, error = merge_or_create_package(package, visit_level=0)
+        if error:
+            break
+
+        if db_package:
+            add_package_to_scan_queue(package=db_package, pipelines=pipelines, priority=priority)
+    return error
+
+
+@priority_router.route("pkg:swift/.*")
+def process_request(purl_str, **kwargs):
+    """
+    Process Swift Package URL (PURL).
+    """
+    from minecode.model_utils import DEFAULT_PIPELINES
+
+    addon_pipelines = kwargs.get("addon_pipelines", [])
+    pipelines = DEFAULT_PIPELINES + tuple(addon_pipelines)
+    priority = kwargs.get("priority", 0)
+
+    package_url = PackageURL.from_string(purl_str)
+    error_msg = map_swift_package(package_url, pipelines, priority)
+
+    if error_msg:
+        return error_msg

minecode/miners/dockerhub.py

@@ -195,3 +195,53 @@ def build_packages_from_jsonfile(metadata, uri=None, purl=None):
         package = scan_models.Package(**common_data)
         package.set_purl(purl)
         yield package
+
+
+def build_package_data(summary, tags_metadata, purl):
+    """
+    Yield ScannedPackage built from PackageData API.
+    """
+
+    namespace = purl.namespace or "library"
+
+    short_desc = summary.get("description")
+    long_desc = summary.get("full_description")
+    descriptions = [d for d in (short_desc, long_desc) if d and d.strip()]
+    description = "\n".join(descriptions)
+    is_private = summary.get("is_private")
+
+    homepage_url = (
+        f"https://hub.docker.com/_/{purl.name}"
+        if namespace == "library"
+        else f"https://hub.docker.com/r/{namespace}/{purl.name}"
+    )
+
+    for tag_metadata in tags_metadata:
+        tag_name = tag_metadata.get("name")
+        size = tag_metadata.get("full_size")
+        digest = tag_metadata.get("digest")
+        sha256 = digest[7::] if digest else None
+
+        last_updater_username = tag_metadata.get("last_updater_username")
+        parties = []
+        if last_updater_username:
+            parties.append(scan_models.Party(name=last_updater_username, role="usernmae"))
+
+        download_data = dict(
+            type="docker",
+            name=purl.name,
+            namespace=purl.namespace,
+            version=purl.version or tag_name,
+            description=description,
+            is_private=is_private,
+            sha256=sha256,
+            parties=parties,
+            size=size,
+            homepage_url=homepage_url,
+            download_url=f"https://hub.docker.com/layers/{namespace}/{purl.name}/{tag_name}/images/{digest}",
+        )
+
+        package = scan_models.PackageData.from_data(download_data)
+        package.datasource_id = "dockerhub_repositories"
+        package.set_purl(purl)
+        yield package