remove file extension filters to match go-minr criteria

Author: coresoftware dev

src/scanoss/file_filters.py

@@ -269,163 +269,6 @@
     'sqlite3',
 }
 
-# TODO: For hfh add the .gitignore patterns
-DEFAULT_SKIPPED_EXT_HFH = {
-    '.1',
-    '.2',
-    '.3',
-    '.4',
-    '.5',
-    '.6',
-    '.7',
-    '.8',
-    '.9',
-    '.ac',
-    '.adoc',
-    '.am',
-    '.asciidoc',
-    '.bmp',
-    '.build',
-    '.cfg',
-    '.chm',
-    '.class',
-    '.cmake',
-    '.cnf',
-    '.conf',
-    '.config',
-    '.contributors',
-    '.copying',
-    '.crt',
-    '.csproj',
-    '.css',
-    '.csv',
-    '.dat',
-    '.data',
-    '.dtd',
-    '.dts',
-    '.iws',
-    '.c9',
-    '.c9revisions',
-    '.dtsi',
-    '.dump',
-    '.eot',
-    '.eps',
-    '.geojson',
-    '.gif',
-    '.glif',
-    '.gmo',
-    '.guess',
-    '.hex',
-    '.htm',
-    '.html',
-    '.ico',
-    '.iml',
-    '.in',
-    '.inc',
-    '.info',
-    '.ini',
-    '.ipynb',
-    '.jpeg',
-    '.jpg',
-    '.json',
-    '.jsonld',
-    '.lock',
-    '.log',
-    '.m4',
-    '.map',
-    '.md5',
-    '.meta',
-    '.mk',
-    '.mxml',
-    '.o',
-    '.otf',
-    '.out',
-    '.pbtxt',
-    '.pdf',
-    '.pem',
-    '.phtml',
-    '.plist',
-    '.png',
-    '.prefs',
-    '.properties',
-    '.pyc',
-    '.qdoc',
-    '.result',
-    '.rgb',
-    '.rst',
-    '.scss',
-    '.sha',
-    '.sha1',
-    '.sha2',
-    '.sha256',
-    '.sln',
-    '.spec',
-    '.sub',
-    '.svg',
-    '.svn-base',
-    '.tab',
-    '.template',
-    '.test',
-    '.tex',
-    '.tiff',
-    '.ttf',
-    '.txt',
-    '.utf-8',
-    '.vim',
-    '.wav',
-    '.woff',
-    '.woff2',
-    '.xht',
-    '.xhtml',
-    '.xml',
-    '.xpm',
-    '.xsd',
-    '.xul',
-    '.yaml',
-    '.yml',
-    '.wfp',
-    '.editorconfig',
-    '.dotcover',
-    '.pid',
-    '.lcov',
-    '.egg',
-    '.manifest',
-    '.cache',
-    '.coverage',
-    '.cover',
-    '.gem',
-    '.lst',
-    '.pickle',
-    '.pdb',
-    '.gml',
-    '.pot',
-    '.plt',
-    '.whml',
-    '.pom',
-    '.smtml',
-    '.min.js',
-    '.mf',
-    '.base64',
-    '.s',
-    '.diff',
-    '.patch',
-    '.rules',
-    # File endings
-    '-doc',
-    'config',
-    'news',
-    'readme',
-    'swiftdoc',
-    'texidoc',
-    'todo',
-    'version',
-    'ignore',
-    'manifest',
-    'sqlite',
-    'sqlite3',
-}
-
-
 class FileFilters(ScanossBase):
     """
     Filter for determining which files to process during scanning, fingerprinting, etc.
@@ -709,7 +552,7 @@ def _should_skip_file(self, file_rel_path: str) -> bool:  # noqa: PLR0911
         file_name = os.path.basename(file_rel_path)
 
         DEFAULT_SKIPPED_FILES_LIST = DEFAULT_SKIPPED_FILES_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_FILES
-        DEFAULT_SKIPPED_EXT_LIST = DEFAULT_SKIPPED_EXT_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT
+        DEFAULT_SKIPPED_EXT_LIST = {} if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT
 
         if not self.hidden_files_folders and file_name.startswith('.'):
             self.print_debug(f'Skipping file: {file_rel_path} (hidden file)')

src/scanoss/scanners/folder_hasher.py

@@ -16,8 +16,6 @@
 
 MINIMUM_FILE_COUNT = 8
 MINIMUM_CONCATENATED_NAME_LENGTH = 32
-MAXIMUM_FILE_NAME_LENGTH = 32
-
 
 class DirectoryNode:
     """
@@ -152,7 +150,7 @@ def _build_root_node(
         root_node = DirectoryNode(str(root))
 
         all_files = [
-            f for f in root.rglob('*') if f.is_file() and len(f.name.encode('utf-8')) <= MAXIMUM_FILE_NAME_LENGTH
+            f for f in root.rglob('*') if f.is_file()
         ]
         filtered_files = self.file_filters.get_filtered_files_from_files(all_files, str(root))
 
@@ -255,8 +253,6 @@ def _hash_calc(self, node: DirectoryNode) -> dict:
 
         for file in node.files:
             key_str = file.key_str
-            if key_str in processed_hashes:
-                continue
 
             file_name = os.path.basename(file.path)