fix use of mmap

Author: JustinWonjaePark

src/fosslight_source/_parsing_scancode_file_item.py

@@ -52,6 +52,7 @@ def parsing_file_item(scancode_file_list, has_error, path_to_scan, need_matched_
     prev_dir = ""
     prev_dir_value = False
     regex = re.compile(r'licenseref-(\S+)', re.IGNORECASE)
+    find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
 
     if scancode_file_list:
         for file in scancode_file_list:
@@ -73,29 +74,17 @@ def parsing_file_item(scancode_file_list, has_error, path_to_scan, need_matched_
 
                     result_item = ScanItem(file_path)
 
-                    fullpath = os.path.join(path_to_scan, file_path) 
+                    fullpath = os.path.join(path_to_scan, file_path)
 
                     urls = file.get("urls", [])
                     url_list = []
-                    print("!!!!!!!! : ",fullpath)
 
                     if urls:
-                        search_term = "SPDX-PackageDownloadLocation:".encode()
                         with open(fullpath, "r") as f:
-                            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as m:
-                                start = 0
-                                while True:
-                                    # Find the next occurrence of the search term
-                                    start = m.find(search_term, start)
-                                    if start == -1:
-                                        break
-                                    # Extract the line that contains the search term
-                                    line = m[start:].split(b"\n")[0].decode()
-                                    spdx_download_location = re.sub(
-                                        r'.*?SPDX-PackageDownloadLocation: ', '', line)
-                                    url_list.append(spdx_download_location)
-                                    # Move the start position to the end of the line
-                                    start += len(line)
+                            with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mmap_obj:
+                                for word in find_word.findall(mmap_obj):
+                                    url_list.append(word.decode('utf-8'))
+
                     if has_error and "scan_errors" in file:
                         error_msg = file.get("scan_errors", [])
                         if len(error_msg) > 0:

src/fosslight_source/cli.py

@@ -70,7 +70,6 @@ def main():
     parser.add_argument('-c', '--cores', type=int, required=False, default=-1)
     parser.add_argument('--no_correction', action='store_true', required=False)
     parser.add_argument('--correct_fpath', nargs=1, type=str, required=False)
-    # parser.add_argument('-u', '--url', action='store_true', required=False)
 
     args = parser.parse_args()
 
@@ -96,8 +95,6 @@ def main():
     correct_filepath = path_to_scan
     if args.correct_fpath:
         correct_filepath = ''.join(args.correct_fpath)
-    #if args.url:
-        #print_url = True
 
     time_out = args.timeout
     core = args.cores

src/fosslight_source/run_scancode.py

@@ -90,7 +90,8 @@ def run_scan(path_to_scan, output_file_name="",
                             _result_log["Error_files"] = error_msg
                             msg = "Failed to analyze :" + error_msg
                     if "files" in results:
-                        rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"], has_error, path_to_scan, need_license)
+                        rc, result_list, parsing_msg, license_list = parsing_file_item(results["files"],
+                                                                                       has_error, path_to_scan, need_license)
                         if parsing_msg:
                             _result_log["Parsing Log"] = parsing_msg
                         if rc:

tests/test_files/run_scancode.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 # Copyright (c) 2020 LG Electronics Inc.
 # SPDX-License-Identifier: Apache-2.0
+
 # SPDX-PackageDownloadLocation: https://dummy_url_for_test.com
 # The code is not licensed under GPL-2.0.
 

tox.ini

@@ -42,9 +42,6 @@ commands =
   fosslight_source -p tests/test_files -m -j -o test_scan2/
   ls test_scan2/
 
-  fosslight_source -p tests/test_files -m -j -o test_scan3/ -s all -u
-  ls test_scan3/
-
   fosslight_convert -p tests/scancode_raw.json -o test_convert -f opossum
 
   python tests/cli_test.py