Update mine Cran to use grouper function

Move extract_cran_packages to miners/cran.py

Signed-off-by: ziad hany <[email protected]>

Author: ziadhany

minecode_pipelines/miners/cran.py

@@ -6,8 +6,10 @@
 # See https://github.com/aboutcode-org/purldb for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import json
 from pathlib import Path
 import requests
+from packageurl import PackageURL
 
 
 def fetch_cran_db(output_file="cran_db.json") -> Path:
@@ -26,3 +28,39 @@ def fetch_cran_db(output_file="cran_db.json") -> Path:
                 f.write(chunk)
 
     return output_path
+
+
+def extract_cran_packages(json_file_path: str) -> list:
+    """
+    Extract package names and their versions from a CRAN DB JSON file.
+    ex:
+    {
+      "AATtools": {
+      "_id": "AATtools",
+      "_rev": "8-9ebb721d05b946f2b437b49e892c9e8c",
+      "name": "AATtools",
+      "versions": {
+         "0.0.1": {...},
+         "0.0.2": {...},
+         "0.0.3": {...}
+      }
+    }
+    """
+    db_path = Path(json_file_path)
+    if not db_path.exists():
+        raise FileNotFoundError(f"File not found: {db_path}")
+
+    with open(db_path, encoding="utf-8") as f:
+        data = json.load(f)
+
+    for pkg_name, pkg_data in data.items():
+        versions = list(pkg_data.get("versions", {}).keys())
+        purls = []
+        for version in versions:
+            purl = PackageURL(
+                type="cran",
+                name=pkg_name,
+                version=version,
+            )
+            purls.append(purl.to_string())
+        yield purls

minecode_pipelines/pipelines/mine_cran.py

@@ -45,6 +45,7 @@ def steps(cls):
             cls.check_federatedcode_eligibility,
             cls.setup_federatedcode_cran,
             cls.mine_and_publish_cran_packageurls,
+            cls.cleanup_db_and_repo,
         )
 
     def check_federatedcode_eligibility(self):

minecode_pipelines/pipes/cran.py

@@ -20,15 +20,13 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/aboutcode-org/scancode.io for support and download.
 
-import json
-from pathlib import Path
-from packageurl import PackageURL
 from aboutcode.hashid import get_package_purls_yml_file_path, get_core_purl
-
 from scanpipe.pipes.federatedcode import commit_changes
 from scanpipe.pipes.federatedcode import push_changes
 from minecode_pipelines import VERSION
+from minecode_pipelines.miners.cran import extract_cran_packages
 from minecode_pipelines.pipes import write_data_to_yaml_file
+from minecode_pipelines.utils import grouper
 
 PACKAGE_BATCH_SIZE = 1000
 
@@ -38,25 +36,32 @@ def mine_and_publish_cran_packageurls(cloned_data_repo, db_path, logger):
     Extract CRAN packages from the database, write their package URLs (purls) to YAML,
     and commit changes in batches to the given cloned repository.
     """
-    batch_counter = 0
-    purl_files = []
-    base_purls = []
+    packages_to_sync = list(extract_cran_packages(db_path))
+
+    for package_batch in grouper(packages_to_sync, PACKAGE_BATCH_SIZE):
+        purl_files = []
+        base_purls = []
 
-    for updated_purls in extract_cran_packages(db_path):
-        batch_counter += 1
-        if not updated_purls:
-            continue
+        if logger:
+            logger(f"Starting package mining for a batch of {PACKAGE_BATCH_SIZE} packages")
 
-        first_purl = updated_purls[0]
-        base_purl = get_core_purl(first_purl)
-        purl_yaml_path = cloned_data_repo.working_dir / get_package_purls_yml_file_path(first_purl)
-        write_data_to_yaml_file(path=purl_yaml_path, data=updated_purls)
+        for updated_purls in package_batch:
+            if not updated_purls:
+                continue  # skip padded None values or empty
+
+            first_purl = updated_purls[0]
+            base_purl = get_core_purl(first_purl)
+            purl_yaml_path = cloned_data_repo.working_dir / get_package_purls_yml_file_path(
+                first_purl
+            )
+            write_data_to_yaml_file(path=purl_yaml_path, data=updated_purls)
 
-        logger(f"writing packageURLs for package: {str(base_purl)} at: {purl_yaml_path}")
-        purl_files.append(purl_yaml_path)
-        base_purls.append(str(base_purl))
+            logger(f"writing packageURLs for package: {str(base_purl)} at: {purl_yaml_path}")
+            purl_files.append(purl_yaml_path)
+            base_purls.append(str(base_purl))
 
-        if purl_files and base_purls and batch_counter > PACKAGE_BATCH_SIZE:
+        # After finishing the batch, commit & push if there’s something to save
+        if purl_files and base_purls:
             commit_changes(
                 repo=cloned_data_repo,
                 files_to_commit=purl_files,
@@ -66,42 +71,3 @@ def mine_and_publish_cran_packageurls(cloned_data_repo, db_path, logger):
                 tool_version=VERSION,
             )
             push_changes(repo=cloned_data_repo)
-
-            batch_counter = 0
-            purl_files.clear()
-            base_purls.clear()
-
-    if purl_files and base_purls:
-        commit_changes(
-            repo=cloned_data_repo,
-            files_to_commit=purl_files,
-            purls=base_purls,
-            mine_type="packageURL",
-            tool_name="pkg:pypi/minecode-pipelines",
-            tool_version=VERSION,
-        )
-        push_changes(repo=cloned_data_repo)
-
-
-def extract_cran_packages(json_file_path: str) -> list:
-    """
-    Extract package names and their versions from a CRAN DB JSON file.
-    """
-    db_path = Path(json_file_path)
-    if not db_path.exists():
-        raise FileNotFoundError(f"File not found: {db_path}")
-
-    with open(db_path, encoding="utf-8") as f:
-        data = json.load(f)
-
-    for pkg_name, pkg_data in data.items():
-        versions = list(pkg_data.get("versions", {}).keys())
-        purls = []
-        for version in versions:
-            purl = PackageURL(
-                type="cran",
-                name=pkg_name,
-                version=version,
-            )
-            purls.append(purl.to_string())
-        yield purls