diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b73157..2262567 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,17 @@ Release notes ============= +Version 32.0.0 - (2024-09-05) +----------------------------- + +- Add new optional argument to generate YAML test data files from a template +- Migrate URLs to new aboutcode.org org +- Drop support for Python 3.7 +- Replace charset_normalizer by chardet because of unstable behavior between minor versions + See https://github.com/jawah/charset_normalizer/issues/520 +- Adopt black and isort style + + Version 31.2.1 - (2024-05-16) ----------------------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 59ab431..d5510e6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -21,7 +21,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -29,7 +29,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macOS-12 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -37,7 +37,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -45,7 +45,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -53,7 +53,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -65,7 +65,7 @@ jobs: parameters: job_name: ubuntu20_test_all_supported_click_versions image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: click_versions: | for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; @@ -82,18 +82,20 @@ jobs: parameters: job_name: ubuntu20_cpython_latest_from_pip image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: - all: - venv/bin/pip install --upgrade-strategy eager --force-reinstall - --upgrade -e . && venv/bin/pytest -n 2 -vvs + all: | + venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . + venv/bin/pip freeze + venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython_latest_from_pip image_name: windows-2019 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: - all: - venv\Scripts\pip install --upgrade-strategy eager --force-reinstall - --upgrade -e . && venv\Scripts\pytest -n 2 -vvs + all: | + venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . + venv\Scripts\pip freeze + venv\Scripts\pytest -n 2 -vvs diff --git a/requirements.txt b/requirements.txt index 2be5ed2..e4b25aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ attrs==22.2.0 beautifulsoup4==4.11.2 certifi==2022.12.7 -charset-normalizer==3.0.1 +chardet==5.2.0 click==8.1.3 idna==3.4 pip==23.0 diff --git a/setup.cfg b/setup.cfg index d7361f8..163d21b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,13 +37,13 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = attrs >= 18.1, !=20.1.0 - Beautifulsoup4 >= 4.0.0 + Beautifulsoup4[chardet] >= 4.0.0 click >= 6.7, !=7.0 - requests >= 2.7.0 + requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/src/commoncode/__init__.py b/src/commoncode/__init__.py index 29888e4..79584fc 100644 --- a/src/commoncode/__init__.py +++ b/src/commoncode/__init__.py @@ -14,16 +14,16 @@ def set_re_max_cache(max_cache=1000000): libraries use a lot of regexes: therefore 100 is not enough to benefit from caching. """ - import re import fnmatch + import re - remax = getattr(re, '_MAXCACHE', 0) + remax = getattr(re, "_MAXCACHE", 0) if remax < max_cache: - setattr(re, '_MAXCACHE', max_cache) + setattr(re, "_MAXCACHE", max_cache) - fnmatchmax = getattr(fnmatch, '_MAXCACHE', 0) + fnmatchmax = getattr(fnmatch, "_MAXCACHE", 0) if fnmatchmax < max_cache: - setattr(fnmatch, '_MAXCACHE', max_cache) + setattr(fnmatch, "_MAXCACHE", max_cache) set_re_max_cache() diff --git a/src/commoncode/archive.py b/src/commoncode/archive.py index 671c451..3c0c758 100644 --- a/src/commoncode/archive.py +++ b/src/commoncode/archive.py @@ -6,12 +6,12 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from functools import partial -import os -from os import path import gzip +import os import tarfile import zipfile +from functools import partial +from os import path from commoncode.system import on_windows @@ -47,7 +47,7 @@ def extract_tar(location, target_dir, verbatim=False, *args, **kwargs): # always for using bytes for paths on all OSses... tar seems to use bytes internally # and get confused otherwise location = os.fsencode(location) - with open(location, 'rb') as input_tar: + with open(location, "rb") as input_tar: tar = None try: tar = tarfile.open(fileobj=input_tar) @@ -69,7 +69,7 @@ def extract_zip(location, target_dir, *args, **kwargs): Extract a zip archive file at location in the target_dir directory. """ if not path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: for info in zipf.infolist(): @@ -82,7 +82,7 @@ def extract_zip(location, target_dir, *args, **kwargs): if not path.exists(target): os.makedirs(target) if not path.exists(target): - with open(target, 'wb') as f: + with open(target, "wb") as f: f.write(content) @@ -92,7 +92,7 @@ def extract_zip_raw(location, target_dir, *args, **kwargs): Use the builtin extractall function """ if not path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: zipf.extractall(path=target_dir) @@ -124,6 +124,6 @@ def get_gz_compressed_file_content(location): Uncompress a compressed file at `location` and return its content as a byte string. Raise Exceptions on errors. """ - with gzip.GzipFile(location, 'rb') as compressed: + with gzip.GzipFile(location, "rb") as compressed: content = compressed.read() return content diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index b58736f..1638088 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -11,11 +11,11 @@ import click -from click.utils import echo -from click.termui import style -from click.types import BoolParamType # FIXME: this is NOT API from click._termui_impl import ProgressBar +from click.termui import style +from click.types import BoolParamType +from click.utils import echo from commoncode.fileutils import file_name from commoncode.fileutils import splitext @@ -31,13 +31,14 @@ def logger_debug(*args): if TRACE: import logging + logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) - and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + """ Command line UI utilities for improved options, help and progress reporting. @@ -51,7 +52,7 @@ class BaseCommand(click.Command): # override this in sub-classes with a command-specific message such as # "Try 'scancode --help' for help on options and arguments." - short_usage_help = '' + short_usage_help = "" def get_usage(self, ctx): """ @@ -61,8 +62,12 @@ def get_usage(self, ctx): return super(BaseCommand, self).get_usage(ctx) + self.short_usage_help def main( - self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra, + self, + args=None, + prog_name=None, + complete_var=None, + standalone_mode=True, + **extra, ): """ Workaround click 4.0 bug https://github.com/mitsuhiko/click/issues/365 @@ -83,14 +88,20 @@ class GroupedHelpCommand(BaseCommand): help where each option is grouped by group in the help. """ - short_usage_help = ''' -Try the '--help' option for help on options and arguments.''' + short_usage_help = """ +Try the '--help' option for help on options and arguments.""" def __init__( - self, name, context_settings=None, callback=None, params=None, + self, + name, + context_settings=None, + callback=None, + params=None, help=None, # NOQA - epilog=None, short_help=None, - options_metavar='[OPTIONS]', add_help_option=True, + epilog=None, + short_help=None, + options_metavar="[OPTIONS]", + add_help_option=True, plugin_options=(), ): """ @@ -120,19 +131,21 @@ def format_options(self, ctx, formatter): to MISC_GROUP group. """ # this mapping defines the CLI help presentation order - help_groups = dict([ - (SCAN_GROUP, []), - (OTHER_SCAN_GROUP, []), - (SCAN_OPTIONS_GROUP, []), - (OUTPUT_GROUP, []), - (OUTPUT_FILTER_GROUP, []), - (OUTPUT_CONTROL_GROUP, []), - (PRE_SCAN_GROUP, []), - (POST_SCAN_GROUP, []), - (CORE_GROUP, []), - (MISC_GROUP, []), - (DOC_GROUP, []), - ]) + help_groups = dict( + [ + (SCAN_GROUP, []), + (OTHER_SCAN_GROUP, []), + (SCAN_OPTIONS_GROUP, []), + (OUTPUT_GROUP, []), + (OUTPUT_FILTER_GROUP, []), + (OUTPUT_CONTROL_GROUP, []), + (PRE_SCAN_GROUP, []), + (POST_SCAN_GROUP, []), + (CORE_GROUP, []), + (MISC_GROUP, []), + (DOC_GROUP, []), + ] + ) for param in self.get_params(ctx): # Get the list of option's name and help text @@ -140,11 +153,11 @@ def format_options(self, ctx, formatter): if not help_record: continue # organize options by group - help_group = getattr(param, 'help_group', MISC_GROUP) - sort_order = getattr(param, 'sort_order', 100) + help_group = getattr(param, "help_group", MISC_GROUP) + sort_order = getattr(param, "sort_order", 100) help_groups[help_group].append((sort_order, help_record)) - with formatter.section('Options'): + with formatter.section("Options"): for group, help_records in help_groups.items(): if not help_records: continue @@ -156,7 +169,6 @@ def format_options(self, ctx, formatter): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 class DebuggedProgressBar(ProgressBar): - # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -221,7 +233,7 @@ def format_progress_line(self): if self.item_show_func: item_info = self.item_show_func(self.current_item) else: - item_info = '.' + item_info = "." if item_info: return item_info @@ -230,16 +242,16 @@ def render_finish(self): BAR_WIDTH = 20 -BAR_SEP_LEN = len(' ') +BAR_SEP_LEN = len(" ") def progressmanager( iterable=None, length=None, - fill_char='#', - empty_char='-', + fill_char="#", + empty_char="-", bar_template=None, - info_sep=' ', + info_sep=" ", show_eta=False, show_percent=False, show_pos=True, @@ -264,10 +276,7 @@ def progressmanager( progress_class = ProgressLogger else: progress_class = EnhancedProgressBar - bar_template = ( - '[%(bar)s]' + ' ' + '%(info)s' - if bar_template is None else bar_template - ) + bar_template = "[%(bar)s]" + " " + "%(info)s" if bar_template is None else bar_template kwargs = dict( iterable=iterable, @@ -293,8 +302,8 @@ def progressmanager( # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 # Here we create a dummy progress_class and then for the attribute presence. pb = progress_class([]) - if hasattr(pb, 'update_min_steps'): - kwargs['update_min_steps'] = update_min_steps + if hasattr(pb, "update_min_steps"): + kwargs["update_min_steps"] = update_min_steps return progress_class(**kwargs) @@ -317,7 +326,7 @@ def fixed_width_file_name(path, max_length=25): >>> assert fwfn == '' """ if not path: - return '' + return "" # get the path as unicode for display! filename = file_name(path) @@ -329,13 +338,13 @@ def fixed_width_file_name(path, max_length=25): remaining_length = max_length - len_ext - dots if remaining_length < 5 or remaining_length < (len_ext + dots): - return '' + return "" prefix_and_suffix_length = abs(remaining_length // 2) prefix = base_name[:prefix_and_suffix_length] - ellipsis = dots * '.' + ellipsis = dots * "." suffix = base_name[-prefix_and_suffix_length:] - return '{prefix}{ellipsis}{suffix}{ext}'.format(**locals()) + return "{prefix}{ellipsis}{suffix}{ext}".format(**locals()) def file_name_max_len(used_width=BAR_WIDTH + 1 + 7 + 1 + 8 + 1): @@ -359,13 +368,13 @@ def file_name_max_len(used_width=BAR_WIDTH + 1 + 7 + 1 + 8 + 1): return max_filename_length -def path_progress_message(item, verbose=False, prefix='Scanned: '): +def path_progress_message(item, verbose=False, prefix="Scanned: "): """ Return a styled message suitable for progress display when processing a path for an `item` tuple of (location, rid, scan_errors, *other items) """ if not item: - return '' + return "" location = item[0] errors = item[2] location = toascii(location) @@ -374,25 +383,25 @@ def path_progress_message(item, verbose=False, prefix='Scanned: '): max_file_name_len = file_name_max_len() # do not display a file name in progress bar if there is no space available if max_file_name_len <= 10: - return '' + return "" progress_line = fixed_width_file_name(location, max_file_name_len) - color = 'red' if errors else 'green' + color = "red" if errors else "green" return style(prefix) + style(progress_line, fg=color) # CLI help groups -SCAN_GROUP = 'primary scans' -SCAN_OPTIONS_GROUP = 'scan options' -OTHER_SCAN_GROUP = 'other scans' -OUTPUT_GROUP = 'output formats' -OUTPUT_CONTROL_GROUP = 'output control' -OUTPUT_FILTER_GROUP = 'output filters' -PRE_SCAN_GROUP = 'pre-scan' -POST_SCAN_GROUP = 'post-scan' -MISC_GROUP = 'miscellaneous' -DOC_GROUP = 'documentation' -CORE_GROUP = 'core' +SCAN_GROUP = "primary scans" +SCAN_OPTIONS_GROUP = "scan options" +OTHER_SCAN_GROUP = "other scans" +OUTPUT_GROUP = "output formats" +OUTPUT_CONTROL_GROUP = "output control" +OUTPUT_FILTER_GROUP = "output filters" +PRE_SCAN_GROUP = "pre-scan" +POST_SCAN_GROUP = "post-scan" +MISC_GROUP = "miscellaneous" +DOC_GROUP = "documentation" +CORE_GROUP = "core" class PluggableCommandLineOption(click.Option): @@ -431,7 +440,7 @@ def __init__( # a sequence of other option name strings that this option # conflicts with if they are set conflicting_options=(), - **kwargs + **kwargs, ): super(PluggableCommandLineOption, self).__init__( param_decls=param_decls, @@ -446,7 +455,7 @@ def __init__( allow_from_autoenv=allow_from_autoenv, type=type, help=help, - **kwargs + **kwargs, ) self.help_group = help_group @@ -463,9 +472,9 @@ def __repr__(self, *args, **kwargs): conflicting_options = self.conflicting_options return ( - 'PluggableCommandLineOption' % locals() + "PluggableCommandLineOption" % locals() ) def validate_dependencies(self, ctx, value): @@ -473,10 +482,8 @@ def validate_dependencies(self, ctx, value): Validate `value` against declared `required_options` or `conflicting_options` dependencies. """ - _validate_option_dependencies( - ctx, self, value, self.required_options, required=True) - _validate_option_dependencies( - ctx, self, value, self.conflicting_options, required=False) + _validate_option_dependencies(ctx, self, value, self.required_options, required=True) + _validate_option_dependencies(ctx, self, value, self.conflicting_options, required=False) def get_help_record(self, ctx): if not self.hidden: @@ -490,22 +497,20 @@ def validate_option_dependencies(ctx): """ values = ctx.params if TRACE: - logger_debug('validate_option_dependencies: values:') + logger_debug("validate_option_dependencies: values:") for va in sorted(values.items()): - logger_debug(' ', va) + logger_debug(" ", va) for param in ctx.command.params: if param.is_eager: continue if not isinstance(param, PluggableCommandLineOption): if TRACE: - logger_debug( - ' validate_option_dependencies: skip param:', param) + logger_debug(" validate_option_dependencies: skip param:", param) continue value = values.get(param.name) if TRACE: - logger_debug(' validate_option_dependencies: param:', - param, 'value:', value) + logger_debug(" validate_option_dependencies: param:", param, "value:", value) param.validate_dependencies(ctx, value) @@ -537,8 +542,8 @@ def _is_set(_value, _param): if TRACE: logger_debug() - logger_debug('Checking param:', param) - logger_debug(' value:', value, 'is_set:', is_set) + logger_debug("Checking param:", param) + logger_debug(" value:", value, "is_set:", is_set) if not is_set: return @@ -556,27 +561,29 @@ def _is_set(_value, _param): if TRACE: logger_debug() - logger_debug(' Available other params:') + logger_debug(" Available other params:") for oparam in oparams: - logger_debug(' other param:', oparam) - logger_debug(' value:', ctx.params.get(oparam.name)) + logger_debug(" other param:", oparam) + logger_debug(" value:", ctx.params.get(oparam.name)) if required: - logger_debug(' missing names:', missing_onames) + logger_debug(" missing names:", missing_onames) if required and missing_onames: opt = param.opts[-1] oopts = [oparam.opts[-1] for oparam in oparams] - omopts = ['--' + oname.replace('_', '-') for oname in missing_onames] + omopts = ["--" + oname.replace("_", "-") for oname in missing_onames] oopts.extend(omopts) - oopts = ', '.join(oopts) - msg = ('The option %(opt)s requires the option(s) %(all_opts)s.' - 'and is missing %(omopts)s. ' - 'You must set all of these options if you use this option.' % locals()) + oopts = ", ".join(oopts) + msg = ( + "The option %(opt)s requires the option(s) %(all_opts)s." + "and is missing %(omopts)s. " + "You must set all of these options if you use this option." % locals() + ) raise click.UsageError(msg) if TRACE: logger_debug() - logger_debug(' Checking other params:') + logger_debug(" Checking other params:") opt = param.opts[-1] @@ -585,21 +592,25 @@ def _is_set(_value, _param): ois_set = _is_set(ovalue, oparam) if TRACE: - logger_debug(' Checking oparam:', oparam) - logger_debug(' value:', ovalue, 'ois_set:', ois_set) + logger_debug(" Checking oparam:", oparam) + logger_debug(" value:", ovalue, "ois_set:", ois_set) # by convention the last opt is the long form oopt = oparam.opts[-1] - oopts = ', '.join(oparam.opts[-1] for oparam in oparams) - all_opts = '%(opt)s and %(oopts)s' % locals() + oopts = ", ".join(oparam.opts[-1] for oparam in oparams) + all_opts = "%(opt)s and %(oopts)s" % locals() if required and not ois_set: - msg = ('The option %(opt)s requires the option(s) %(oopts)s ' - 'and is missing %(oopt)s. ' - 'You must set all of these options if you use this option.' % locals()) + msg = ( + "The option %(opt)s requires the option(s) %(oopts)s " + "and is missing %(oopt)s. " + "You must set all of these options if you use this option." % locals() + ) raise click.UsageError(msg) if not required and ois_set: - msg = ('The option %(opt)s cannot be used together with the %(oopts)s option(s) ' - 'and %(oopt)s is used. ' - 'You can set only one of these options at a time.' % locals()) + msg = ( + "The option %(opt)s cannot be used together with the %(oopts)s option(s) " + "and %(oopt)s is used. " + "You can set only one of these options at a time." % locals() + ) raise click.UsageError(msg) diff --git a/src/commoncode/codec.py b/src/commoncode/codec.py index 0707068..fe84d63 100644 --- a/src/commoncode/codec.py +++ b/src/commoncode/codec.py @@ -14,8 +14,12 @@ """ -def c2i(c): return c -def i2c(i): return bytes([i]) +def c2i(c): + return c + + +def i2c(i): + return bytes([i]) def num_to_bin(num): @@ -26,16 +30,16 @@ def num_to_bin(num): """ # Zero is not encoded but returned as an empty value if num == 0: - return b'\x00' + return b"\x00" - return num.to_bytes((num.bit_length() + 7) // 8, 'big') + return num.to_bytes((num.bit_length() + 7) // 8, "big") def bin_to_num(binstr): """ Convert a big endian byte-ordered binary string to an integer or long. """ - return int.from_bytes(binstr, byteorder='big', signed=False) + return int.from_bytes(binstr, byteorder="big", signed=False) def urlsafe_b64encode(s): @@ -50,7 +54,7 @@ def urlsafe_b64decode(b64): Decode a url safe base64-encoded string. Note that we use stddecode to work around a bug in the standard library. """ - b = b64.replace(b'-', b'+').replace(b'_', b'/') + b = b64.replace(b"-", b"+").replace(b"_", b"/") return stddecode(b) diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 053e7fb..2570a05 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -6,20 +6,19 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import ctypes import contextlib +import ctypes import io -import os -from os import path - import logging +import os import signal import subprocess +from os import path +from commoncode import text from commoncode.fileutils import get_temp_dir from commoncode.system import on_posix from commoncode.system import on_windows -from commoncode import text """ Wrapper for executing external commands in sub-processes which works @@ -33,15 +32,16 @@ if TRACE: import sys + logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) # current directory is the root dir of this library curr_dir = path.dirname(path.dirname(path.abspath(__file__))) -PATH_ENV_VAR = 'PATH' -LD_LIBRARY_PATH = 'LD_LIBRARY_PATH' -DYLD_LIBRARY_PATH = 'DYLD_LIBRARY_PATH' +PATH_ENV_VAR = "PATH" +LD_LIBRARY_PATH = "LD_LIBRARY_PATH" +DYLD_LIBRARY_PATH = "DYLD_LIBRARY_PATH" def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): @@ -68,10 +68,10 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): cwd = cwd or curr_dir # temp files for stderr and stdout - tmp_dir = get_temp_dir(prefix='cmd-') + tmp_dir = get_temp_dir(prefix="cmd-") - sop = path.join(tmp_dir, 'stdout') - sep = path.join(tmp_dir, 'stderr') + sop = path.join(tmp_dir, "stdout") + sep = path.join(tmp_dir, "stderr") # shell==True is DANGEROUS but we are not running arbitrary commands # though we can execute commands that just happen to be in the path @@ -81,15 +81,15 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): if log: printer = logger.debug if TRACE else lambda x: print(x) printer( - 'Executing command %(cmd_loc)r as:\n%(full_cmd)r\nwith: env=%(env)r\n' - 'shell=%(shell)r\ncwd=%(cwd)r\nstdout=%(sop)r\nstderr=%(sep)r' - % locals()) + "Executing command %(cmd_loc)r as:\n%(full_cmd)r\nwith: env=%(env)r\n" + "shell=%(shell)r\ncwd=%(cwd)r\nstdout=%(sop)r\nstderr=%(sep)r" % locals() + ) proc = None rc = 100 try: - with io.open(sop, 'wb') as stdout, io.open(sep, 'wb') as stderr, pushd(cmd_dir): + with io.open(sop, "wb") as stdout, io.open(sep, "wb") as stderr, pushd(cmd_dir): proc = subprocess.Popen( full_cmd, cwd=cwd, @@ -108,11 +108,11 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): if not to_files: # return output as ASCII string loaded from the output files - with open(sop, 'rb') as so: + with open(sop, "rb") as so: sor = so.read() sop = text.toascii(sor).strip() - with open(sep, 'rb') as se: + with open(sep, "rb") as se: ser = se.read() sep = text.toascii(ser).strip() @@ -167,18 +167,15 @@ def get_env(base_vars=None, lib_dir=None): # Create and add LD environment variables if lib_dir and on_posix: - new_path = f'{lib_dir}' + new_path = f"{lib_dir}" # on Linux/posix ld_lib_path = os.environ.get(LD_LIBRARY_PATH) - env_vars.update( - {LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) + env_vars.update({LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) # on Mac, though LD_LIBRARY_PATH should work too dyld_lib_path = os.environ.get(DYLD_LIBRARY_PATH) - env_vars.update( - {DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) + env_vars.update({DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) - env_vars = {text.as_unicode(k): text.as_unicode(v) - for k, v in env_vars.items()} + env_vars = {text.as_unicode(k): text.as_unicode(v) for k, v in env_vars.items()} return env_vars @@ -198,9 +195,9 @@ def close_pipe(p): except IOError: pass - close_pipe(getattr(proc, 'stdin', None)) - close_pipe(getattr(proc, 'stdout', None)) - close_pipe(getattr(proc, 'stderr', None)) + close_pipe(getattr(proc, "stdin", None)) + close_pipe(getattr(proc, "stdout", None)) + close_pipe(getattr(proc, "stderr", None)) try: # Ensure process death otherwise proc.wait may hang in some cases @@ -219,8 +216,7 @@ def load_shared_library(dll_loc, *args): Return the loaded shared library object from the ``dll_loc`` location. """ if not dll_loc or not path.exists(dll_loc): - raise ImportError( - f'Shared library does not exists: dll_loc: {dll_loc}') + raise ImportError(f"Shared library does not exists: dll_loc: {dll_loc}") if not isinstance(dll_loc, str): dll_loc = os.fsdecode(dll_loc) @@ -232,19 +228,22 @@ def load_shared_library(dll_loc, *args): with pushd(dll_dir): lib = ctypes.CDLL(dll_loc) except OSError as e: - from pprint import pformat import traceback - msgs = tuple([ - f'ctypes.CDLL("{dll_loc}")', - 'os.environ:\n{}'.format(pformat(dict(os.environ))), - traceback.format_exc(), - ]) + from pprint import pformat + + msgs = tuple( + [ + f'ctypes.CDLL("{dll_loc}")', + "os.environ:\n{}".format(pformat(dict(os.environ))), + traceback.format_exc(), + ] + ) raise Exception(msgs) from e if lib and lib._name: return lib - raise Exception(f'Failed to load shared library with ctypes: {dll_loc}') + raise Exception(f"Failed to load shared library with ctypes: {dll_loc}") @contextlib.contextmanager @@ -271,7 +270,7 @@ def update_path_var(existing_path_var, new_path): if not new_path: return existing_path_var - existing_path_var = existing_path_var or '' + existing_path_var = existing_path_var or "" existing_path_var = os.fsdecode(existing_path_var) new_path = os.fsdecode(new_path) @@ -296,7 +295,11 @@ def update_path_var(existing_path_var, new_path): return updated_path_var -PATH_VARS = DYLD_LIBRARY_PATH, LD_LIBRARY_PATH, 'PATH', +PATH_VARS = ( + DYLD_LIBRARY_PATH, + LD_LIBRARY_PATH, + "PATH", +) def searchable_paths(env_vars=PATH_VARS): @@ -306,7 +309,7 @@ def searchable_paths(env_vars=PATH_VARS): """ dirs = [] for env_var in env_vars: - value = os.environ.get(env_var, '') or '' + value = os.environ.get(env_var, "") or "" dirs.extend(value.split(os.pathsep)) dirs = [os.path.realpath(d.strip()) for d in dirs if d.strip()] return tuple(d for d in dirs if os.path.isdir(d)) diff --git a/src/commoncode/compat.py b/src/commoncode/compat.py index 001464c..bad5d38 100644 --- a/src/commoncode/compat.py +++ b/src/commoncode/compat.py @@ -8,4 +8,7 @@ unicode = str # NOQA long = int # NOQA -integer_types = int, long, +integer_types = ( + int, + long, +) diff --git a/src/commoncode/datautils.py b/src/commoncode/datautils.py index ea71f9a..db67e21 100644 --- a/src/commoncode/datautils.py +++ b/src/commoncode/datautils.py @@ -6,22 +6,32 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import typing + import attr from attr.validators import in_ as choices # NOQA -import typing """ Utilities and helpers for data classes. """ -HELP_METADATA = '__field_help' -LABEL_METADATA = '__field_label' +HELP_METADATA = "__field_help" +LABEL_METADATA = "__field_label" -def attribute(default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - init=True, type=None, converter=None, # NOQA - help=None, label=None, metadata=None,): # NOQA +def attribute( + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + init=True, + type=None, + converter=None, # NOQA + help=None, + label=None, + metadata=None, +): # NOQA """ A generic attribute with help metadata and that is not included in the representation by default. @@ -42,12 +52,20 @@ def attribute(default=attr.NOTHING, validator=None, init=init, metadata=metadata, type=type, - converter=converter + converter=converter, ) -def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Boolean( + default=False, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A boolean attribute. """ @@ -65,8 +83,16 @@ def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # N ) -def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def TriBoolean( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A tri-boolean attribute with possible values of None, True and False. """ @@ -84,8 +110,16 @@ def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # ) -def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def String( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A string attribute. """ @@ -103,8 +137,16 @@ def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQ ) -def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Integer( + default=0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ An integer attribute. """ @@ -123,8 +165,16 @@ def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Float( + default=0.0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A float attribute. """ @@ -142,9 +192,17 @@ def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def List(item_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def List( + item_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A list attribute: the optional item_type defines the type of items it stores. """ @@ -165,9 +223,17 @@ def List(item_type=typing.Any, default=attr.NOTHING, validator=None, ) -def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, help=None, label=None): # NOQA +def Mapping( + value_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + help=None, + label=None, +): # NOQA """ A mapping attribute: the optional value_type defines the type of values it stores. The key is always a string. @@ -191,13 +257,22 @@ def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, label=label, ) + ################################################## # FIXME: add proper support for dates!!! ################################################## -def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Date( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A date attribute. It always serializes to an ISO date string. Behavior is TBD and for now this is exactly a string. diff --git a/src/commoncode/date.py b/src/commoncode/date.py index 81da3d0..2493265 100644 --- a/src/commoncode/date.py +++ b/src/commoncode/date.py @@ -7,12 +7,12 @@ # import calendar -from datetime import datetime import os +from datetime import datetime def isoformat(utc_date): - return datetime.isoformat(utc_date).replace('T', ' ') + return datetime.isoformat(utc_date).replace("T", " ") def get_file_mtime(location, iso=True): @@ -20,7 +20,7 @@ def get_file_mtime(location, iso=True): Return a string containing the last modified date of a file formatted as an ISO time stamp if ISO is True or as a raw number since epoch. """ - date = '' + date = "" # FIXME: use file types if not os.path.isdir(location): mtime = os.stat(location).st_mtime @@ -37,5 +37,4 @@ def secs_from_epoch(d): Return a number of seconds since epoch for a date time stamp """ # FIXME: what does this do? - return calendar.timegm(datetime.strptime(d.split('.')[0], - '%Y-%m-%d %H:%M:%S').timetuple()) + return calendar.timegm(datetime.strptime(d.split(".")[0], "%Y-%m-%d %H:%M:%S").timetuple()) diff --git a/src/commoncode/dict_utils.py b/src/commoncode/dict_utils.py index ca017d5..1473f6d 100644 --- a/src/commoncode/dict_utils.py +++ b/src/commoncode/dict_utils.py @@ -1,4 +1,3 @@ - # Copyright (c) 2003-2012 Raymond Hettinger # SPDX-License-Identifier: Python-2.0 diff --git a/src/commoncode/distro.py b/src/commoncode/distro.py index 3b22354..2e1a29a 100644 --- a/src/commoncode/distro.py +++ b/src/commoncode/distro.py @@ -32,11 +32,5 @@ def parse_os_release(location): """ with open(location) as osrl: lines = (line.strip() for line in osrl) - lines = ( - line.partition('=') for line in lines - if line and not line.startswith('#') - ) - return { - key.strip(): ''.join(shlex.split(value)) - for key, _, value in lines - } + lines = (line.partition("=") for line in lines if line and not line.startswith("#")) + return {key.strip(): "".join(shlex.split(value)) for key, _, value in lines} diff --git a/src/commoncode/fetch.py b/src/commoncode/fetch.py index 9428e32..3db98c2 100644 --- a/src/commoncode/fetch.py +++ b/src/commoncode/fetch.py @@ -35,19 +35,18 @@ def download_url(url, file_name=None, verify=True, timeout=10): try: response = requests.get(url, **requests_args) except (ConnectionError, InvalidSchema) as e: - logger.error('download_url: Download failed for %(url)r' % locals()) + logger.error("download_url: Download failed for %(url)r" % locals()) raise status = response.status_code if status != 200: - msg = 'download_url: Download failed for %(url)r with %(status)r' % locals( - ) + msg = "download_url: Download failed for %(url)r with %(status)r" % locals() logger.error(msg) raise Exception(msg) - tmp_dir = fileutils.get_temp_dir(prefix='fetch-') + tmp_dir = fileutils.get_temp_dir(prefix="fetch-") output_file = os.path.join(tmp_dir, file_name) - with open(output_file, 'wb') as out: + with open(output_file, "wb") as out: out.write(response.content) return output_file diff --git a/src/commoncode/fileset.py b/src/commoncode/fileset.py index 99323f8..9ebab26 100644 --- a/src/commoncode/fileset.py +++ b/src/commoncode/fileset.py @@ -83,16 +83,14 @@ def is_included(path, includes=None, excludes=None): if includes: included = get_matches(path, includes, all_matches=False) if TRACE: - logger.debug( - 'in_fileset: path: %(path)r included:%(included)r' % locals()) + logger.debug("in_fileset: path: %(path)r included:%(included)r" % locals()) if not included: return False if excludes: excluded = get_matches(path, excludes, all_matches=False) if TRACE: - logger.debug( - 'in_fileset: path: %(path)r excluded:%(excluded)r .' % locals()) + logger.debug("in_fileset: path: %(path)r excluded:%(excluded)r ." % locals()) if excluded: return False @@ -110,41 +108,39 @@ def get_matches(path, patterns, all_matches=False): return False path = fileutils.as_posixpath(path).lower() - pathstripped = path.lstrip('/0') + pathstripped = path.lstrip("/0") if not pathstripped: return False segments = paths.split(pathstripped) if TRACE: - logger.debug( - '_match: path: %(path)r patterns:%(patterns)r.' % locals()) + logger.debug("_match: path: %(path)r patterns:%(patterns)r." % locals()) matches = [] if not isinstance(patterns, dict): - assert isinstance(patterns, (list, tuple) - ), 'Invalid patterns: {}'.format(patterns) + assert isinstance(patterns, (list, tuple)), "Invalid patterns: {}".format(patterns) patterns = {p: p for p in patterns} for pat, value in patterns.items(): if not pat or not pat.strip(): continue - value = value or '' - pat = pat.lstrip('/').lower() - is_plain = '/' not in pat + value = value or "" + pat = pat.lstrip("/").lower() + is_plain = "/" not in pat if is_plain: if any(fnmatch.fnmatchcase(s, pat) for s in segments): matches.append(value) if not all_matches: break - elif (fnmatch.fnmatchcase(path, pat) or fnmatch.fnmatchcase(pathstripped, pat)): + elif fnmatch.fnmatchcase(path, pat) or fnmatch.fnmatchcase(pathstripped, pat): matches.append(value) if not all_matches: break if TRACE: - logger.debug('_match: matches: %(matches)r' % locals()) + logger.debug("_match: matches: %(matches)r" % locals()) if not all_matches: if matches: @@ -161,9 +157,9 @@ def load(location): if not location: return tuple() fn = os.path.abspath(os.path.normpath(os.path.expanduser(location))) - msg = ('File %(location)s does not exist or not a file.') % locals() - assert (os.path.exists(fn) and os.path.isfile(fn)), msg - mode = 'r' + msg = ("File %(location)s does not exist or not a file.") % locals() + assert os.path.exists(fn) and os.path.isfile(fn), msg + mode = "r" with open(fn, mode) as f: return [l.strip() for l in f if l and l.strip()] @@ -175,9 +171,9 @@ def includes_excludes(patterns, message): value in the returned mappings. Ignore pattern as comments if prefixed with #. Use an empty string is message is None. """ - message = message or '' - BANG = '!' - POUND = '#' + message = message or "" + BANG = "!" + POUND = "#" included = {} excluded = {} if not patterns: diff --git a/src/commoncode/filetype.py b/src/commoncode/filetype.py index 04d8210..a7681d4 100644 --- a/src/commoncode/filetype.py +++ b/src/commoncode/filetype.py @@ -7,10 +7,11 @@ # import os -from datetime import datetime, timezone +from datetime import datetime +from datetime import timezone -from commoncode.system import on_posix from commoncode.functional import memoize +from commoncode.system import on_posix """ Low level file type utilities, essentially a wrapper around os.path and stat. @@ -77,7 +78,7 @@ def get_link_target(location): Return the link target for `location` if this is a Link or an empty string. """ - target = '' + target = "" # always false on windows, until Python supports junctions/links if on_posix and is_link(location): try: @@ -91,12 +92,38 @@ def get_link_target(location): # Map of type checker function -> short type code # The order of types check matters: link -> file -> directory -> special -TYPES = dict([ - (is_link, ('l', 'link',)), - (is_file, ('f', 'file',)), - (is_dir, ('d', 'directory',)), - (is_special, ('s', 'special',)) -]) +TYPES = dict( + [ + ( + is_link, + ( + "l", + "link", + ), + ), + ( + is_file, + ( + "f", + "file", + ), + ), + ( + is_dir, + ( + "d", + "directory", + ), + ), + ( + is_special, + ( + "s", + "special", + ), + ), + ] +) def get_type(location, short=True): @@ -161,7 +188,7 @@ def get_last_modified_date(location): Return the last modified date stamp of a file as YYYYMMDD format. The date of non-files (dir, links, special) is always an empty string. """ - yyyymmdd = '' + yyyymmdd = "" if is_file(location): utc_date = datetime.isoformat( datetime.fromtimestamp( @@ -174,8 +201,8 @@ def get_last_modified_date(location): counting_functions = { - 'file_count': lambda _: 1, - 'file_size': os.path.getsize, + "file_count": lambda _: 1, + "file_size": os.path.getsize, } @@ -205,8 +232,9 @@ def counter(location, counting_function): count_fun = counting_functions[counting_function] return count_fun(location) elif is_dir(location): - count += sum(counter(os.path.join(location, p), counting_function) - for p in os.listdir(location)) + count += sum( + counter(os.path.join(location, p), counting_function) for p in os.listdir(location) + ) return count @@ -216,7 +244,7 @@ def get_file_count(location): or 1 if `location` is a file. Only regular files are counted. Everything else has a zero size. """ - return counter(location, 'file_count') + return counter(location, "file_count") def get_size(location): @@ -225,4 +253,4 @@ def get_size(location): directory, the cumulative size of all files in this directory tree. Only regular files have a size. Everything else has a zero size. """ - return counter(location, 'file_size') + return counter(location, "file_size") diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index d0aa38b..123a717 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -7,14 +7,13 @@ # import errno -import os import ntpath +import os import posixpath import shutil import stat import sys import tempfile - from os import fsdecode try: @@ -33,7 +32,9 @@ class WindowsError(Exception): pass + import logging + logger = logging.getLogger(__name__) TRACE = False @@ -48,7 +49,8 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + """ File, paths and directory utility functions. @@ -68,8 +70,7 @@ def create_dir(location): if os.path.exists(location): if not os.path.isdir(location): - err = ('Cannot create directory: existing file ' - 'in the way ''%(location)s.') + err = "Cannot create directory: existing file " "in the way " "%(location)s." raise OSError(err % locals()) else: # may fail on win if the path is too long @@ -96,7 +97,7 @@ def create_dir(location): raise -def get_temp_dir(base_dir=_base_temp_dir, prefix=''): +def get_temp_dir(base_dir=_base_temp_dir, prefix=""): """ Return the path to a new existing unique temporary directory, created under the `base_dir` base directory using the `prefix` prefix. @@ -108,7 +109,7 @@ def get_temp_dir(base_dir=_base_temp_dir, prefix=''): has_base = bool(base_dir) if not has_base: - base_dir = os.getenv('SCANCODE_TMP') + base_dir = os.getenv("SCANCODE_TMP") if not base_dir: base_dir = tempfile.gettempdir() @@ -116,10 +117,11 @@ def get_temp_dir(base_dir=_base_temp_dir, prefix=''): create_dir(base_dir) if not has_base: - prefix = 'scancode-tk-' + prefix = "scancode-tk-" return tempfile.mkdtemp(prefix=prefix, dir=base_dir) + # # PATHS AND NAMES MANIPULATIONS # @@ -145,8 +147,8 @@ def is_posixpath(location): Return False if the `location` path is likely a Windows-like path using backslash as path separators (e.g. "\"). """ - has_slashes = '/' in location - has_backslashes = '\\' in location + has_slashes = "/" in location + has_backslashes = "\\" in location # windows paths with drive if location: drive, _ = ntpath.splitdrive(location) @@ -168,7 +170,7 @@ def as_posixpath(location): accepts gracefully POSIX paths on Windows. """ location = prepare_path(location) - return location.replace('\\', '/') + return location.replace("\\", "/") def as_winpath(location): @@ -177,7 +179,7 @@ def as_winpath(location): `location` path. """ location = prepare_path(location) - return location.replace('/', '\\') + return location.replace("/", "\\") def split_parent_resource(path, force_posix=False): @@ -186,7 +188,7 @@ def split_parent_resource(path, force_posix=False): """ use_posix = force_posix or is_posixpath(path) splitter = use_posix and posixpath or ntpath - path_no_trailing_speps = path.rstrip('\\/') + path_no_trailing_speps = path.rstrip("\\/") return splitter.split(path_no_trailing_speps) @@ -196,7 +198,7 @@ def resource_name(path, force_posix=False): is the last path segment. """ _left, right = split_parent_resource(path, force_posix) - return right or '' + return right or "" def file_name(path, force_posix=False): @@ -213,8 +215,8 @@ def parent_directory(path, force_posix=False, with_trail=True): """ left, _right = split_parent_resource(path, force_posix) use_posix = force_posix or is_posixpath(path) - sep = '/' if use_posix else '\\' - trail = sep if with_trail and left != sep else '' + sep = "/" if use_posix else "\\" + trail = sep if with_trail and left != sep else "" return left + trail @@ -241,19 +243,19 @@ def splitext_name(file_name, is_file=True): """ if not file_name: - return '', '' + return "", "" file_name = fsdecode(file_name) if not is_file: - return file_name, '' + return file_name, "" - if file_name.startswith('.') and '.' not in file_name[1:]: + if file_name.startswith(".") and "." not in file_name[1:]: # .dot files base name is the full name and they do not have an extension - return file_name, '' + return file_name, "" base_name, extension = posixpath.splitext(file_name) # handle composed extensions of tar.gz, bz, zx,etc - if base_name.endswith('.tar'): + if base_name.endswith(".tar"): base_name, extension2 = posixpath.splitext(base_name) extension = extension2 + extension return base_name, extension @@ -266,30 +268,36 @@ def splitext(path, force_posix=False): the file name minus its extension. Return an empty extension string for a directory. """ - base_name = '' - extension = '' + base_name = "" + extension = "" if not path: return base_name, extension - is_dir = path.endswith(('\\', '/',)) - path = as_posixpath(path).strip('/') + is_dir = path.endswith( + ( + "\\", + "/", + ) + ) + path = as_posixpath(path).strip("/") name = resource_name(path, force_posix) if is_dir: # directories never have an extension base_name = name - extension = '' - elif name.startswith('.') and '.' not in name[1:]: + extension = "" + elif name.startswith(".") and "." not in name[1:]: # .dot files base name is the full name and they do not have an extension base_name = name - extension = '' + extension = "" else: base_name, extension = posixpath.splitext(name) # handle composed extensions of tar.gz, tar.bz2, zx,etc - if base_name.endswith('.tar'): + if base_name.endswith(".tar"): base_name, extension2 = posixpath.splitext(base_name) extension = extension2 + extension return base_name, extension + # # DIRECTORY AND FILES WALKING/ITERATION # @@ -318,7 +326,7 @@ def walk(location, ignored=None, follow_symlinks=False): is_ignored = ignored(location) if ignored else False if is_ignored: if TRACE: - logger_debug('walk: ignored:', location, is_ignored) + logger_debug("walk: ignored:", location, is_ignored) return if filetype.is_file(location, follow_symlinks=follow_symlinks): @@ -331,14 +339,16 @@ def walk(location, ignored=None, follow_symlinks=False): for name in os.listdir(location): loc = os.path.join(location, name) if filetype.is_special(loc) or (ignored and ignored(loc)): - if (follow_symlinks - and filetype.is_link(loc) - and not filetype.is_broken_link(location)): + if ( + follow_symlinks + and filetype.is_link(loc) + and not filetype.is_broken_link(location) + ): pass else: if TRACE: ign = ignored and ignored(loc) - logger_debug('walk: ignored:', loc, ign) + logger_debug("walk: ignored:", loc, ign) continue # special files and symlinks are always ignored if filetype.is_dir(loc, follow_symlinks=follow_symlinks): @@ -348,7 +358,9 @@ def walk(location, ignored=None, follow_symlinks=False): yield location, dirs, files for dr in dirs: - for tripple in walk(os.path.join(location, dr), ignored, follow_symlinks=follow_symlinks): + for tripple in walk( + os.path.join(location, dr), ignored, follow_symlinks=follow_symlinks + ): yield tripple @@ -367,6 +379,8 @@ def resource_iter(location, ignored=ignore_nothing, with_dirs=True, follow_symli yield os.path.join(top, d) for f in files: yield os.path.join(top, f) + + # # COPY # @@ -450,7 +464,7 @@ def copytime(src, dst): """ errors = [] st = os.stat(src) - if hasattr(os, 'utime'): + if hasattr(os, "utime"): try: os.utime(dst, (st.st_atime, st.st_mtime)) except OSError as why: @@ -461,6 +475,7 @@ def copytime(src, dst): errors.append((src, dst, str(why))) return errors + # # PERMISSIONS # @@ -516,6 +531,7 @@ def chmod_tree(location, flags): for f in files: chmod(os.path.join(top, f), flags, recurse=False) + # # DELETION # @@ -527,7 +543,7 @@ def _rm_handler(function, path, excinfo): # NOQA This retries deleting once before giving up. """ if TRACE: - logger_debug('_rm_handler:', 'path:', path, 'excinfo:', excinfo) + logger_debug("_rm_handler:", "path:", path, "excinfo:", excinfo) if function in (os.rmdir, os.listdir): try: chmod(path, RW, recurse=True) @@ -536,7 +552,7 @@ def _rm_handler(function, path, excinfo): # NOQA pass if os.path.exists(path): - logger.warning('Failed to delete directory %s', path) + logger.warning("Failed to delete directory %s", path) elif function == os.remove: try: @@ -545,7 +561,7 @@ def _rm_handler(function, path, excinfo): # NOQA pass if os.path.exists(path): - logger.warning('Failed to delete file %s', path) + logger.warning("Failed to delete file %s", path) def delete(location, _err_handler=_rm_handler): diff --git a/src/commoncode/functional.py b/src/commoncode/functional.py index 8bb6f64..eaa7376 100644 --- a/src/commoncode/functional.py +++ b/src/commoncode/functional.py @@ -7,9 +7,8 @@ # import functools - -from types import GeneratorType from array import array +from types import GeneratorType def flatten(seq): @@ -87,8 +86,7 @@ def memoized(*args, **kwargs): if kwargs: return fun(*args, **kwargs) # convert any list args to a tuple - args = tuple(tuple(arg) if isinstance(arg, (list, tuple, array)) else arg - for arg in args) + args = tuple(tuple(arg) if isinstance(arg, (list, tuple, array)) else arg for arg in args) try: return memos[args] except KeyError: diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 61be375..954d3f4 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -11,9 +11,9 @@ import sys from functools import partial +from commoncode import filetype from commoncode.codec import bin_to_num from commoncode.codec import urlsafe_b64encode -from commoncode import filetype """ Hashes and checksums. @@ -33,19 +33,18 @@ def _hash_mod(bitsize, hmodule): """ class hasher(object): - def __init__(self, msg=None): self.digest_size = bitsize // 8 - self.h = msg and hmodule(msg).digest()[:self.digest_size] or None + self.h = msg and hmodule(msg).digest()[: self.digest_size] or None def digest(self): return bytes(self.h) def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode('utf-8') + return self.h and binascii.hexlify(self.h).decode("utf-8") def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode('utf-8') + return self.h and urlsafe_b64encode(self.h).decode("utf-8") def intdigest(self): return self.h and int(bin_to_num(self.h)) @@ -72,7 +71,7 @@ def intdigest(self): 160: _hash_mod(160, hashlib.sha1), 256: _hash_mod(256, hashlib.sha256), 384: _hash_mod(384, hashlib.sha384), - 512: _hash_mod(512, hashlib.sha512) + 512: _hash_mod(512, hashlib.sha512), } @@ -94,29 +93,29 @@ def __init__(self, msg=None): def _compute(self, msg): # note: bytes interpolation is new in Python 3.5 - git_blob_msg = b'blob %d\0%s' % (len(msg), msg) + git_blob_msg = b"blob %d\0%s" % (len(msg), msg) return hashlib.sha1(git_blob_msg).digest() def digest(self): return bytes(self.h) def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode('utf-8') + return self.h and binascii.hexlify(self.h).decode("utf-8") def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode('utf-8') + return self.h and urlsafe_b64encode(self.h).decode("utf-8") def intdigest(self): return self.h and int(bin_to_num(self.h)) _hashmodules_by_name = { - 'md5': get_hasher(128), - 'sha1': get_hasher(160), - 'sha1_git': sha1_git_hasher, - 'sha256': get_hasher(256), - 'sha384': get_hasher(384), - 'sha512': get_hasher(512) + "md5": get_hasher(128), + "sha1": get_hasher(160), + "sha1_git": sha1_git_hasher, + "sha256": get_hasher(256), + "sha384": get_hasher(384), + "sha512": get_hasher(512), } @@ -131,7 +130,7 @@ def checksum(location, name, base64=False): hasher = _hashmodules_by_name[name] # fixme: we should read in chunks? - with open(location, 'rb') as f: + with open(location, "rb") as f: hashable = f.read() hashed = hasher(hashable) @@ -142,30 +141,30 @@ def checksum(location, name, base64=False): def md5(location): - return checksum(location, name='md5', base64=False) + return checksum(location, name="md5", base64=False) def sha1(location): - return checksum(location, name='sha1', base64=False) + return checksum(location, name="sha1", base64=False) def b64sha1(location): - return checksum(location, name='sha1', base64=True) + return checksum(location, name="sha1", base64=True) def sha256(location): - return checksum(location, name='sha256', base64=False) + return checksum(location, name="sha256", base64=False) def sha512(location): - return checksum(location, name='sha512', base64=False) + return checksum(location, name="sha512", base64=False) def sha1_git(location): - return checksum(location, name='sha1_git', base64=False) + return checksum(location, name="sha1_git", base64=False) -def multi_checksums(location, checksum_names=('md5', 'sha1', 'sha256', 'sha512', 'sha1_git')): +def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", "sha1_git")): """ Return a mapping of hexdigest checksums keyed by checksum name from the content of the file at `location`. Use the `checksum_names` list of checksum names. @@ -177,7 +176,7 @@ def multi_checksums(location, checksum_names=('md5', 'sha1', 'sha256', 'sha512', return results # fixme: we should read in chunks? - with open(location, 'rb') as f: + with open(location, "rb") as f: hashable = f.read() for name in checksum_names: diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index dc21d67..c19ccdc 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -38,10 +38,7 @@ def is_ignore_file(location): """ Return True if the location is an ignore file. """ - return ( - filetype.is_file(location) - and fileutils.file_name(location) == '.scancodeignore' - ) + return filetype.is_file(location) and fileutils.file_name(location) == ".scancodeignore" def get_ignores(location, include_defaults=True): @@ -59,244 +56,236 @@ def get_ignores(location, include_defaults=True): unignores.update(uni) return ignores, unignores + # # Default ignores # ignores_MacOSX = { - '.DS_Store': 'Default ignore: MacOSX artifact', - '._.DS_Store': 'Default ignore: MacOSX artifact', - '__MACOSX': 'Default ignore: MacOSX artifact', - '.AppleDouble': 'Default ignore: MacOSX artifact', - '.LSOverride': 'Default ignore: MacOSX artifact', - '.DocumentRevisions-V100': 'Default ignore: MacOSX artifact', - '.fseventsd': 'Default ignore: MacOSX artifact', - '.Spotlight-V100': 'Default ignore: MacOSX artifact', - '.VolumeIcon.icns': 'Default ignore: MacOSX artifact', - - '.journal': 'Default ignore: MacOSX DMG/HFS+ artifact', - '.journal_info_block': 'Default ignore: MacOSX DMG/HFS+ artifact', - '.Trashes': 'Default ignore: MacOSX DMG/HFS+ artifact', - r'\[HFS+ Private Data\]': 'Default ignore: MacOSX DMG/HFS+ artifact private data', + ".DS_Store": "Default ignore: MacOSX artifact", + "._.DS_Store": "Default ignore: MacOSX artifact", + "__MACOSX": "Default ignore: MacOSX artifact", + ".AppleDouble": "Default ignore: MacOSX artifact", + ".LSOverride": "Default ignore: MacOSX artifact", + ".DocumentRevisions-V100": "Default ignore: MacOSX artifact", + ".fseventsd": "Default ignore: MacOSX artifact", + ".Spotlight-V100": "Default ignore: MacOSX artifact", + ".VolumeIcon.icns": "Default ignore: MacOSX artifact", + ".journal": "Default ignore: MacOSX DMG/HFS+ artifact", + ".journal_info_block": "Default ignore: MacOSX DMG/HFS+ artifact", + ".Trashes": "Default ignore: MacOSX DMG/HFS+ artifact", + r"\[HFS+ Private Data\]": "Default ignore: MacOSX DMG/HFS+ artifact private data", } ignores_Windows = { - 'Thumbs.db': 'Default ignore: Windows artifact', - 'ehthumbs.db': 'Default ignore: Windows artifact', - 'Desktop.ini': 'Default ignore: Windows artifact', - '$RECYCLE.BIN': 'Default ignore: Windows artifact', - '*.lnk': 'Default ignore: Windows artifact', - 'System Volume Information': 'Default ignore: Windows FS artifact', - 'NTUSER.DAT*': 'Default ignore: Windows FS artifact', + "Thumbs.db": "Default ignore: Windows artifact", + "ehthumbs.db": "Default ignore: Windows artifact", + "Desktop.ini": "Default ignore: Windows artifact", + "$RECYCLE.BIN": "Default ignore: Windows artifact", + "*.lnk": "Default ignore: Windows artifact", + "System Volume Information": "Default ignore: Windows FS artifact", + "NTUSER.DAT*": "Default ignore: Windows FS artifact", } ignores_Linux = { - '.directory': 'Default ignore: KDE artifact', - '.Trash-*': 'Default ignore: Linux/Gome/KDE artifact', + ".directory": "Default ignore: KDE artifact", + ".Trash-*": "Default ignore: Linux/Gome/KDE artifact", } ignores_IDEs = { - '*.el': 'Default ignore: EMACS Elisp artifact', - '*.swp': 'Default ignore: VIM artifact', - '.project': 'Default ignore: Eclipse IDE artifact', - '.pydevproject': 'Default ignore: Eclipse IDE artifact', - '.settings': 'Default ignore: Eclipse IDE artifact', - '.eclipse': 'Default ignore: Eclipse IDE artifact', - '.loadpath': 'Default ignore: Eclipse IDE artifact', - '*.launch': 'Default ignore: Eclipse IDE artifact', - '.cproject': 'Default ignore: Eclipse IDE artifact', - '.cdtproject': 'Default ignore: Eclipse IDE artifact', - '.classpath': 'Default ignore: Eclipse IDE artifact', - '.buildpath': 'Default ignore: Eclipse IDE artifact', - '.texlipse': 'Default ignore: Eclipse IDE artifact', - - '*.iml': 'Default ignore: JetBrains IDE artifact', - '*.ipr': 'Default ignore: JetBrains IDE artifact', - '*.iws': 'Default ignore: JetBrains IDE artifact', - '.idea/': 'Default ignore: JetBrains IDE artifact', - '.idea_modules/': 'Default ignore: JetBrains IDE artifact', - - '*.kdev4': 'Default ignore: Kdevelop artifact', - '.kdev4/': 'Default ignore: Kdevelop artifact', - - '*.nib': 'Default ignore: Apple Xcode artifact', - '*.plst': 'Default ignore: Apple Xcode plist artifact', - '*.pbxuser': 'Default ignore: Apple Xcode artifact', - '*.pbxproj': 'Default ignore: Apple Xcode artifact', - 'xcuserdata': 'Default ignore: Apple Xcode artifact', - '*.xcuserstate': 'Default ignore: Apple Xcode artifact', - - '*.csproj': 'Default ignore: Microsoft VS project artifact', - '*.unityproj': 'Default ignore: Microsoft VS project artifact', - '*.sln': 'Default ignore: Microsoft VS project artifact', - '*.sluo': 'Default ignore: Microsoft VS project artifact', - '*.suo': 'Default ignore: Microsoft VS project artifact', - '*.user': 'Default ignore: Microsoft VS project artifact', - '*.sln.docstates': 'Default ignore: Microsoft VS project artifact', - '*.dsw': 'Default ignore: Microsoft VS project artifact', - - '.editorconfig': 'Default ignore: Editor config artifact', - - ' Leiningen.gitignore': 'Default ignore: Leiningen artifact', - '.architect': 'Default ignore: ExtJS artifact', - '*.tmproj': 'Default ignore: Textmate artifact', - '*.tmproject': 'Default ignore: Textmate artifact', + "*.el": "Default ignore: EMACS Elisp artifact", + "*.swp": "Default ignore: VIM artifact", + ".project": "Default ignore: Eclipse IDE artifact", + ".pydevproject": "Default ignore: Eclipse IDE artifact", + ".settings": "Default ignore: Eclipse IDE artifact", + ".eclipse": "Default ignore: Eclipse IDE artifact", + ".loadpath": "Default ignore: Eclipse IDE artifact", + "*.launch": "Default ignore: Eclipse IDE artifact", + ".cproject": "Default ignore: Eclipse IDE artifact", + ".cdtproject": "Default ignore: Eclipse IDE artifact", + ".classpath": "Default ignore: Eclipse IDE artifact", + ".buildpath": "Default ignore: Eclipse IDE artifact", + ".texlipse": "Default ignore: Eclipse IDE artifact", + "*.iml": "Default ignore: JetBrains IDE artifact", + "*.ipr": "Default ignore: JetBrains IDE artifact", + "*.iws": "Default ignore: JetBrains IDE artifact", + ".idea/": "Default ignore: JetBrains IDE artifact", + ".idea_modules/": "Default ignore: JetBrains IDE artifact", + "*.kdev4": "Default ignore: Kdevelop artifact", + ".kdev4/": "Default ignore: Kdevelop artifact", + "*.nib": "Default ignore: Apple Xcode artifact", + "*.plst": "Default ignore: Apple Xcode plist artifact", + "*.pbxuser": "Default ignore: Apple Xcode artifact", + "*.pbxproj": "Default ignore: Apple Xcode artifact", + "xcuserdata": "Default ignore: Apple Xcode artifact", + "*.xcuserstate": "Default ignore: Apple Xcode artifact", + "*.csproj": "Default ignore: Microsoft VS project artifact", + "*.unityproj": "Default ignore: Microsoft VS project artifact", + "*.sln": "Default ignore: Microsoft VS project artifact", + "*.sluo": "Default ignore: Microsoft VS project artifact", + "*.suo": "Default ignore: Microsoft VS project artifact", + "*.user": "Default ignore: Microsoft VS project artifact", + "*.sln.docstates": "Default ignore: Microsoft VS project artifact", + "*.dsw": "Default ignore: Microsoft VS project artifact", + ".editorconfig": "Default ignore: Editor config artifact", + " Leiningen.gitignore": "Default ignore: Leiningen artifact", + ".architect": "Default ignore: ExtJS artifact", + "*.tmproj": "Default ignore: Textmate artifact", + "*.tmproject": "Default ignore: Textmate artifact", } ignores_web = { - '.htaccess': 'Default ignore: .htaccess file', - 'robots.txt': 'Default ignore: robots file', - 'humans.txt': 'Default ignore: robots file', - 'web.config': 'Default ignore: web config', - '.htaccess.sample': 'Default ignore: .htaccess file', + ".htaccess": "Default ignore: .htaccess file", + "robots.txt": "Default ignore: robots file", + "humans.txt": "Default ignore: robots file", + "web.config": "Default ignore: web config", + ".htaccess.sample": "Default ignore: .htaccess file", } ignores_Maven = { - 'pom.xml.tag': 'Default ignore: Maven artifact', - 'pom.xml.releaseBackup': 'Default ignore: Maven artifact', - 'pom.xml.versionsBackup': 'Default ignore: Maven artifact', - 'pom.xml.next': 'Default ignore: Maven artifact', - 'release.properties': 'Default ignore: Maven artifact', - 'dependency-reduced-pom.xml': 'Default ignore: Maven artifact', - 'buildNumber.properties': 'Default ignore: Maven artifact', + "pom.xml.tag": "Default ignore: Maven artifact", + "pom.xml.releaseBackup": "Default ignore: Maven artifact", + "pom.xml.versionsBackup": "Default ignore: Maven artifact", + "pom.xml.next": "Default ignore: Maven artifact", + "release.properties": "Default ignore: Maven artifact", + "dependency-reduced-pom.xml": "Default ignore: Maven artifact", + "buildNumber.properties": "Default ignore: Maven artifact", } ignores_VCS = { - '.bzr': 'Default ignore: Bazaar artifact', - '.bzrignore': 'Default ignore: Bazaar config artifact', - - '.git': 'Default ignore: Git artifact', - '.gitignore': 'Default ignore: Git config artifact', - '.gitattributes': 'Default ignore: Git config artifact', - - '.hg': 'Default ignore: Mercurial artifact', - '.hgignore': 'Default ignore: Mercurial config artifact', - - '.repo': 'Default ignore: Multiple Git repository artifact', - - '.svn': 'Default ignore: SVN artifact', - '.svnignore': 'Default ignore: SVN config artifact', - - '.tfignore': 'Default ignore: Microsft TFS config artifact', - - 'vssver.scc': 'Default ignore: Visual Source Safe artifact', - - 'CVS': 'Default ignore: CVS artifact', - '.cvsignore': 'Default ignore: CVS config artifact', - - '*/_MTN': 'Default ignore: Monotone artifact', - '*/_darcs': 'Default ignore: Darcs artifact', - '*/{arch}': 'Default ignore: GNU Arch artifact', + ".bzr": "Default ignore: Bazaar artifact", + ".bzrignore": "Default ignore: Bazaar config artifact", + ".git": "Default ignore: Git artifact", + ".gitignore": "Default ignore: Git config artifact", + ".gitattributes": "Default ignore: Git config artifact", + ".hg": "Default ignore: Mercurial artifact", + ".hgignore": "Default ignore: Mercurial config artifact", + ".repo": "Default ignore: Multiple Git repository artifact", + ".svn": "Default ignore: SVN artifact", + ".svnignore": "Default ignore: SVN config artifact", + ".tfignore": "Default ignore: Microsft TFS config artifact", + "vssver.scc": "Default ignore: Visual Source Safe artifact", + "CVS": "Default ignore: CVS artifact", + ".cvsignore": "Default ignore: CVS config artifact", + "*/_MTN": "Default ignore: Monotone artifact", + "*/_darcs": "Default ignore: Darcs artifact", + "*/{arch}": "Default ignore: GNU Arch artifact", } ignores_Medias = { - 'pspbrwse.jbf': 'Default ignore: Paintshop browse file', - 'Thumbs.db': 'Default ignore: Image thumbnails DB', - 'Thumbs.db:encryptable': 'Default ignore: Image thumbnails DB', - 'thumbs/': 'Default ignore: Image thumbnails DB', - '_thumbs/': 'Default ignore: Image thumbnails DB', + "pspbrwse.jbf": "Default ignore: Paintshop browse file", + "Thumbs.db": "Default ignore: Image thumbnails DB", + "Thumbs.db:encryptable": "Default ignore: Image thumbnails DB", + "thumbs/": "Default ignore: Image thumbnails DB", + "_thumbs/": "Default ignore: Image thumbnails DB", } ignores_Build_scripts = { - 'Makefile.in': 'Default ignore: automake artifact', - 'Makefile.am': 'Default ignore: automake artifact', - 'autom4te.cache': 'Default ignore: autoconf artifact', - '*.m4': 'Default ignore: autotools artifact', - 'configure': 'Default ignore: Configure script', - 'configure.bat': 'Default ignore: Configure script', - 'configure.sh': 'Default ignore: Configure script', - 'configure.ac': 'Default ignore: Configure script', - 'config.guess': 'Default ignore: Configure script', - 'config.sub': 'Default ignore: Configure script', - 'compile': 'Default ignore: autoconf artifact', - 'depcomp': 'Default ignore: autoconf artifact', - 'ltmain.sh': 'Default ignore: libtool autoconf artifact', - 'install-sh': 'Default ignore: autoconf artifact', - 'missing': 'Default ignore: autoconf artifact', - 'mkinstalldirs': 'Default ignore: autoconf artifact', - 'stamp-h1': 'Default ignore: autoconf artifact', - 'm4/': 'Default ignore: autoconf artifact', - 'autogen.sh': 'Default ignore: autotools artifact', - 'autogen.sh': 'Default ignore: autotools artifact', - - 'CMakeCache.txt': 'Default ignore: CMake artifact', - 'cmake_install.cmake': 'Default ignore: CMake artifact', - 'install_manifest.txt': 'Default ignore: CMake artifact', + "Makefile.in": "Default ignore: automake artifact", + "Makefile.am": "Default ignore: automake artifact", + "autom4te.cache": "Default ignore: autoconf artifact", + "*.m4": "Default ignore: autotools artifact", + "configure": "Default ignore: Configure script", + "configure.bat": "Default ignore: Configure script", + "configure.sh": "Default ignore: Configure script", + "configure.ac": "Default ignore: Configure script", + "config.guess": "Default ignore: Configure script", + "config.sub": "Default ignore: Configure script", + "compile": "Default ignore: autoconf artifact", + "depcomp": "Default ignore: autoconf artifact", + "ltmain.sh": "Default ignore: libtool autoconf artifact", + "install-sh": "Default ignore: autoconf artifact", + "missing": "Default ignore: autoconf artifact", + "mkinstalldirs": "Default ignore: autoconf artifact", + "stamp-h1": "Default ignore: autoconf artifact", + "m4/": "Default ignore: autoconf artifact", + "autogen.sh": "Default ignore: autotools artifact", + "autogen.sh": "Default ignore: autotools artifact", + "CMakeCache.txt": "Default ignore: CMake artifact", + "cmake_install.cmake": "Default ignore: CMake artifact", + "install_manifest.txt": "Default ignore: CMake artifact", } ignores_CI = { - '.travis.yml': 'Default ignore: Travis config', - '.coveragerc': 'Default ignore: Coverall config', + ".travis.yml": "Default ignore: Travis config", + ".coveragerc": "Default ignore: Coverall config", } ignores_Python = { - 'pip-selfcheck.json': 'Default ignore: Pip workfile', - 'pytest.ini': 'Default ignore: Python pytest config', - 'tox.ini': 'Default ignore: Python tox config', - '__pycache__/': 'Default ignore: Python bytecode cache', - '.installed.cfg': 'Default ignore: Python Buildout artifact', - 'pip-log.txt': 'Default ignore: Python pip artifact', - 'pip-delete-this-directory.txt': 'Default ignore: Python pip artifact', - 'pyvenv.cfg': 'Default ignore: Python virtualenv artifact', + "pip-selfcheck.json": "Default ignore: Pip workfile", + "pytest.ini": "Default ignore: Python pytest config", + "tox.ini": "Default ignore: Python tox config", + "__pycache__/": "Default ignore: Python bytecode cache", + ".installed.cfg": "Default ignore: Python Buildout artifact", + "pip-log.txt": "Default ignore: Python pip artifact", + "pip-delete-this-directory.txt": "Default ignore: Python pip artifact", + "pyvenv.cfg": "Default ignore: Python virtualenv artifact", } ignores_I18N = { - '*.mo': 'Default ignore: Translation file', - '*.pot': 'Default ignore: Translation file', - '.localized': 'Default ignore: localized file', + "*.mo": "Default ignore: Translation file", + "*.pot": "Default ignore: Translation file", + ".localized": "Default ignore: localized file", } ignores_coverage_and_tests = { - '*.gcno': 'Default ignore: GCC coverage', - '*.gcda': 'Default ignore: GCC coverage', - '*.gcov': 'Default ignore: GCC coverage', - '.last_cover_stats': 'Default ignore: Perl coverage', - 'htmlcov/': 'Default ignore: Python coverage', - '.tox/': 'Default ignore: Tox tem dir', - '.coverage': 'Default ignore: Python coverage', - '.coverage.*': 'Default ignore: Python coverage', - 'nosetests.xml': 'Default ignore: Python nose tests', - 'coverage.xml': 'Default ignore: Python coverage', - '/spec/reports/': 'Default ignore: Ruby Rails test report', - '/rdoc/': 'Default ignore: Ruby doc', - '.rvmrc': 'Default ignore: Ruby RVM', - '.sass-cache': 'Default ignore: Saas cache', - '*.css.map': 'Default ignore: Saas map', - 'phpunit.xml': 'Default ignore: phpunit', - '*.VisualState.xml': 'Default ignore: Nunit', - 'TestResult.xml': 'Default ignore: Nunit', + "*.gcno": "Default ignore: GCC coverage", + "*.gcda": "Default ignore: GCC coverage", + "*.gcov": "Default ignore: GCC coverage", + ".last_cover_stats": "Default ignore: Perl coverage", + "htmlcov/": "Default ignore: Python coverage", + ".tox/": "Default ignore: Tox tem dir", + ".coverage": "Default ignore: Python coverage", + ".coverage.*": "Default ignore: Python coverage", + "nosetests.xml": "Default ignore: Python nose tests", + "coverage.xml": "Default ignore: Python coverage", + "/spec/reports/": "Default ignore: Ruby Rails test report", + "/rdoc/": "Default ignore: Ruby doc", + ".rvmrc": "Default ignore: Ruby RVM", + ".sass-cache": "Default ignore: Saas cache", + "*.css.map": "Default ignore: Saas map", + "phpunit.xml": "Default ignore: phpunit", + "*.VisualState.xml": "Default ignore: Nunit", + "TestResult.xml": "Default ignore: Nunit", } ignores_Misc = { - 'pax_global_header': 'Default ignore: Pax header file', - 'C++.gitignore': 'Default ignore: C++.gitignore', - '.gwt/': 'Default ignore: GWT compilation logs', - '.gwt-tmp/': 'Default ignore: GWT temp files', - 'gradle-app.setting': 'Default ignore: Graddle app settings', - 'hs_err_pid*': 'Default ignore: Java VM crash logs', - '.grunt': 'Default ignore: Grunt intermediate storage', - '.history': 'Default ignore: History file', - '.~lock.*#': 'Default ignore: LibreOffice locks', - '/.ssh': 'Default ignore: SSH configuration', + "pax_global_header": "Default ignore: Pax header file", + "C++.gitignore": "Default ignore: C++.gitignore", + ".gwt/": "Default ignore: GWT compilation logs", + ".gwt-tmp/": "Default ignore: GWT temp files", + "gradle-app.setting": "Default ignore: Graddle app settings", + "hs_err_pid*": "Default ignore: Java VM crash logs", + ".grunt": "Default ignore: Grunt intermediate storage", + ".history": "Default ignore: History file", + ".~lock.*#": "Default ignore: LibreOffice locks", + "/.ssh": "Default ignore: SSH configuration", } default_ignores = {} -default_ignores.update(chain(*[d.items() for d in [ - ignores_MacOSX, - ignores_Windows, - ignores_Linux, - ignores_IDEs, - ignores_web, - ignores_Maven, - ignores_VCS, - ignores_Medias, - ignores_Build_scripts, - ignores_CI, - ignores_Python, - ignores_I18N, - ignores_coverage_and_tests, - ignores_Misc, - ignores_Build_scripts, -]])) +default_ignores.update( + chain( + *[ + d.items() + for d in [ + ignores_MacOSX, + ignores_Windows, + ignores_Linux, + ignores_IDEs, + ignores_web, + ignores_Maven, + ignores_VCS, + ignores_Medias, + ignores_Build_scripts, + ignores_CI, + ignores_Python, + ignores_I18N, + ignores_coverage_and_tests, + ignores_Misc, + ignores_Build_scripts, + ] + ] + ) +) diff --git a/src/commoncode/paths.py b/src/commoncode/paths.py index 8718c50..f9d11d6 100644 --- a/src/commoncode/paths.py +++ b/src/commoncode/paths.py @@ -9,14 +9,13 @@ import ntpath import posixpath import re - from os.path import commonprefix -from commoncode.text import as_unicode -from commoncode.text import toascii from commoncode.fileutils import as_posixpath from commoncode.fileutils import as_winpath from commoncode.fileutils import is_posixpath +from commoncode.text import as_unicode +from commoncode.text import toascii """ Various path utilities such as common prefix and suffix functions, conversion @@ -53,18 +52,15 @@ def safe_path(path, posix=False, preserve_spaces=False, posix_only=False): segments = [s.strip() for s in path.split(path_sep) if s.strip()] segments = [ - portable_filename( - s, - preserve_spaces=preserve_spaces, - posix_only=posix_only - ) for s in segments + portable_filename(s, preserve_spaces=preserve_spaces, posix_only=posix_only) + for s in segments ] if not segments: - return '_' + return "_" # always return posix - path = '/'.join(segments) + path = "/".join(segments) return as_posixpath(path) @@ -78,7 +74,7 @@ def path_handlers(path, posix=True): is_posix = is_posixpath(path) use_posix = posix or is_posix pathmod = use_posix and posixpath or ntpath - path_sep = '/' if use_posix else '\\' + path_sep = "/" if use_posix else "\\" return pathmod, path_sep @@ -94,11 +90,11 @@ def resolve(path, posix=True): Windows path with blackslash separators otherwise. """ if not path: - return '.' + return "." path = path.strip() if not path: - return '.' + return "." if not is_posixpath(path): path = as_winpath(path) @@ -110,7 +106,7 @@ def resolve(path, posix=True): segments = [s.strip() for s in path.split(path_sep) if s.strip()] # remove empty (// or ///) or blank (space only) or single dot segments - segments = [s for s in segments if s and s != '.'] + segments = [s for s in segments if s and s != "."] path = path_sep.join(segments) @@ -123,30 +119,30 @@ def resolve(path, posix=True): segments = [s.strip() for s in segments if s and s.strip()] # is this a windows absolute path? if yes strip the colon to make this relative - if segments and len(segments[0]) == 2 and segments[0].endswith(':'): + if segments and len(segments[0]) == 2 and segments[0].endswith(":"): segments[0] = segments[0][:-1] # replace any remaining (usually leading) .. segment with a literal "dotdot" - dotdot = 'dotdot' - dd = '..' + dotdot = "dotdot" + dd = ".." segments = [dotdot if s == dd else s for s in segments if s] if segments: path = path_sep.join(segments) else: - path = '.' + path = "." path = as_posixpath(path) return path -legal_punctuation = r'!\#$%&\(\)\+,\-\.;\=@\[\]_\{\}\~' -legal_spaces = r' ' -legal_alphanumeric = r'A-Za-z0-9' +legal_punctuation = r"!\#$%&\(\)\+,\-\.;\=@\[\]_\{\}\~" +legal_spaces = r" " +legal_alphanumeric = r"A-Za-z0-9" legal_chars = legal_alphanumeric + legal_punctuation legal_chars_inc_spaces = legal_chars + legal_spaces -illegal_chars_re = r'[^' + legal_chars + r']' -illegal_chars_exc_spaces_re = r'[^' + legal_chars_inc_spaces + r']' +illegal_chars_re = r"[^" + legal_chars + r"]" +illegal_chars_exc_spaces_re = r"[^" + legal_chars_inc_spaces + r"]" replace_illegal_chars = re.compile(illegal_chars_re).sub replace_illegal_chars_exc_spaces = re.compile(illegal_chars_exc_spaces_re).sub @@ -154,18 +150,38 @@ def resolve(path, posix=True): posix_legal_punctuation = r'<:"/>\|\*\^\\\'`\?' + legal_punctuation posix_legal_chars = legal_alphanumeric + posix_legal_punctuation posix_legal_chars_inc_spaces = posix_legal_chars + legal_spaces -posix_illegal_chars_re = r'[^' + posix_legal_chars + r']' -posix_illegal_chars_exc_spaces_re = r'[^' + posix_legal_chars_inc_spaces + r']' +posix_illegal_chars_re = r"[^" + posix_legal_chars + r"]" +posix_illegal_chars_exc_spaces_re = r"[^" + posix_legal_chars_inc_spaces + r"]" replace_illegal_posix_chars = re.compile(posix_illegal_chars_re).sub -replace_illegal_posix_chars_exc_spaces = re.compile( - posix_illegal_chars_exc_spaces_re).sub - - -ILLEGAL_WINDOWS_NAMES = set([ - 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', - 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9', - 'aux', 'con', 'nul', 'prn' -]) +replace_illegal_posix_chars_exc_spaces = re.compile(posix_illegal_chars_exc_spaces_re).sub + + +ILLEGAL_WINDOWS_NAMES = set( + [ + "com1", + "com2", + "com3", + "com4", + "com5", + "com6", + "com7", + "com8", + "com9", + "lpt1", + "lpt2", + "lpt3", + "lpt4", + "lpt5", + "lpt6", + "lpt7", + "lpt8", + "lpt9", + "aux", + "con", + "nul", + "prn", + ] +) def portable_filename(filename, preserve_spaces=False, posix_only=False): @@ -192,32 +208,32 @@ def portable_filename(filename, preserve_spaces=False, posix_only=False): filename = toascii(filename, translit=True) if not filename: - return '_' + return "_" if posix_only: if preserve_spaces: - filename = replace_illegal_posix_chars_exc_spaces('_', filename) + filename = replace_illegal_posix_chars_exc_spaces("_", filename) else: - filename = replace_illegal_posix_chars('_', filename) + filename = replace_illegal_posix_chars("_", filename) else: if preserve_spaces: - filename = replace_illegal_chars_exc_spaces('_', filename) + filename = replace_illegal_chars_exc_spaces("_", filename) else: - filename = replace_illegal_chars('_', filename) + filename = replace_illegal_chars("_", filename) if not posix_only: - basename, dot, extension = filename.partition('.') + basename, dot, extension = filename.partition(".") if basename.lower() in ILLEGAL_WINDOWS_NAMES: - filename = ''.join([basename, '_', dot, extension]) + filename = "".join([basename, "_", dot, extension]) # no name made only of dots. - if set(filename) == set(['.']): - filename = 'dot' * len(filename) + if set(filename) == set(["."]): + filename = "dot" * len(filename) # replaced any leading dotdot - if filename != '..' and filename.startswith('..'): - while filename.startswith('..'): - filename = filename.replace('..', '__', 1) + if filename != ".." and filename.startswith(".."): + while filename.startswith(".."): + filename = filename.replace("..", "__", 1) return filename @@ -233,7 +249,12 @@ def common_prefix(s1, s2): """ if not s1 or not s2: return None, 0 - common = commonprefix((s1, s2,)) + common = commonprefix( + ( + s1, + s2, + ) + ) if common: return common, len(common) else: @@ -276,8 +297,8 @@ def split(p): """ if not p: return [] - p = p.strip('/').split('/') - return [] if p == [''] else p + p = p.strip("/").split("/") + return [] if p == [""] else p def _common_path(p1, p2, common_func): @@ -287,5 +308,5 @@ def _common_path(p1, p2, common_func): function. """ common, lgth = common_func(split(p1), split(p2)) - common = '/'.join(common) if common else None + common = "/".join(common) if common else None return common, lgth diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 2e79948..6ce1536 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -21,9 +21,9 @@ from os.path import isfile from os.path import join from os.path import normpath +from posixpath import dirname as posixpath_parent from posixpath import join as posixpath_join from posixpath import normpath as posixpath_normpath -from posixpath import dirname as posixpath_parent import attr @@ -33,7 +33,7 @@ # alway have something there. import tempfile - temp_dir = tempfile.mkdtemp(prefix='scancode-resource-cache') + temp_dir = tempfile.mkdtemp(prefix="scancode-resource-cache") from commoncode import ignore from commoncode.datautils import List @@ -77,7 +77,7 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) class ResourceNotInCache(Exception): @@ -98,11 +98,11 @@ def skip_ignored(location): if TRACE_DEEP: logger_debug() logger_debug( - 'Codebase.populate: walk: ignored loc:', + "Codebase.populate: walk: ignored loc:", location, - 'ignored:', + "ignored:", ignored(location), - 'is_special:', + "is_special:", is_special(location), ) @@ -157,20 +157,18 @@ class Header(object): should create a Header and append it to the Codebase.headers list. """ - tool_name = String(help='Name of the tool used such as scancode-toolkit.') - tool_version = String(default='', help='Tool version used such as v1.2.3.') - options = Mapping( - help='Mapping of key/values describing the options used with this tool.') - notice = String(default='', help='Notice text for this tool.') - start_timestamp = String(help='Start timestamp for this header.') - end_timestamp = String(help='End timestamp for this header.') - output_format_version = String( - help='Version for the output data format, such as v1.1 .') - duration = String(help='Scan duration in seconds.') - message = String(help='Message text.') - errors = List(help='List of error messages.') - warnings = List(help='List of warning messages.') - extra_data = Mapping(help='Mapping of extra key/values for this tool.') + tool_name = String(help="Name of the tool used such as scancode-toolkit.") + tool_version = String(default="", help="Tool version used such as v1.2.3.") + options = Mapping(help="Mapping of key/values describing the options used with this tool.") + notice = String(default="", help="Notice text for this tool.") + start_timestamp = String(help="Start timestamp for this header.") + end_timestamp = String(help="End timestamp for this header.") + output_format_version = String(help="Version for the output data format, such as v1.1 .") + duration = String(help="Scan duration in seconds.") + message = String(help="Message text.") + errors = List(help="List of error messages.") + warnings = List(help="List of warning messages.") + extra_data = Mapping(help="Mapping of extra key/values for this tool.") def to_dict(self): return attr.asdict(self, dict_factory=dict) @@ -204,28 +202,28 @@ class Codebase: # we do not really need slots but this is a way to ensure we have tight # control on object attributes __slots__ = ( - 'max_depth', - 'location', - 'has_single_resource', - 'resource_attributes', - 'resource_class', - 'root', - 'is_file', - 'temp_dir', - 'resources_by_path', - 'resources_count', - 'paths', - 'max_in_memory', - 'all_in_memory', - 'all_on_disk', - 'cache_dir', - 'headers', - 'current_header', - 'codebase_attributes', - 'attributes', - 'counters', - 'timings', - 'errors', + "max_depth", + "location", + "has_single_resource", + "resource_attributes", + "resource_class", + "root", + "is_file", + "temp_dir", + "resources_by_path", + "resources_count", + "paths", + "max_in_memory", + "all_in_memory", + "all_on_disk", + "cache_dir", + "headers", + "current_header", + "codebase_attributes", + "attributes", + "counters", + "timings", + "errors", ) # the value returned if the resource is cached @@ -286,7 +284,7 @@ def __init__( ######################################################################## location = os.fsdecode(location) location = abspath(normpath(expanduser(location))) - location = location.rstrip('/\\') + location = location.rstrip("/\\") # TODO: what if is_special(location)??? assert exists(location) self.location = location @@ -294,8 +292,7 @@ def __init__( self.is_file = filetype_is_file(location) # True if this codebase root is a file or an empty directory. - self.has_single_resource = bool( - self.is_file or not os.listdir(location)) + self.has_single_resource = bool(self.is_file or not os.listdir(location)) ######################################################################## # Set up caching, summary, timing, and error info @@ -311,8 +308,11 @@ def _prepare_clean_paths(self, paths=tuple()): We convert to POSIX and ensure we have no slash at both ends. """ paths = (clean_path(p) for p in (paths or []) if p) + # we sort by path segments (e.g. essentially a topo sort) - def _sorter(p): return p.split('/') + def _sorter(p): + return p.split("/") + return sorted(paths, key=_sorter) def _setup_essentials(self, temp_dir=temp_dir, max_in_memory=10000): @@ -391,7 +391,7 @@ def _get_resource_cache_location(self, path, create_dirs=False): path = clean_path(path) # for the cached file name, we use an md5 of the path to avoid things being too long - resid = str(md5(path.encode('utf-8')).hexdigest()) + resid = str(md5(path.encode("utf-8")).hexdigest()) cache_sub_dir, cache_file_name = resid[-2:], resid parent = join(self.cache_dir, cache_sub_dir) @@ -412,7 +412,7 @@ def _build_resource_class(self, *args, **kwargs): """ # Resource sub-class to use. Configured with plugin attributes if present return attr.make_class( - name='ScannedResource', + name="ScannedResource", attrs=self.resource_attributes or {}, slots=True, bases=(Resource,), @@ -441,8 +441,7 @@ def _populate(self): # Codebase attributes to use. Configured with plugin attributes if # present. self.codebase_attributes = self._collect_codebase_attributes() - cbac = _CodebaseAttributes.from_attributes( - attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) self.attributes = cbac() # Resource sub-class to use. Configured with plugin attributes if present @@ -456,7 +455,7 @@ def _populate(self): ########################################################## root = self._create_root_resource() if TRACE: - logger_debug('Codebase.populate: root:', root) + logger_debug("Codebase.populate: root:", root) if self.has_single_resource: # there is nothing else to do for a single file or a single @@ -483,7 +482,7 @@ def _create_resources_from_paths(self, root, paths): for path in paths: res_loc = join(base_location, path) if not exists(res_loc): - msg = f'ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}' + msg = f"ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}" self.errors.append(msg) raise Exception(path, join(base_location, path)) continue @@ -491,19 +490,17 @@ def _create_resources_from_paths(self, root, paths): # create all parents. The last parent is the one we want to use parent = root if TRACE: - logger_debug( - 'Codebase._create_resources_from_paths: parent', parent) + logger_debug("Codebase._create_resources_from_paths: parent", parent) for parent_path in get_ancestor_paths(path, include_self=False): if TRACE: logger_debug( - f' Codebase._create_resources_from_paths: parent_path: {parent_path!r}' + f" Codebase._create_resources_from_paths: parent_path: {parent_path!r}" ) if not parent_path: continue newpar = parents_by_path.get(parent_path) if TRACE: - logger_debug( - ' Codebase._create_resources_from_paths: newpar', repr(newpar)) + logger_debug(" Codebase._create_resources_from_paths: newpar", repr(newpar)) if not newpar: newpar = self._get_or_create_resource( @@ -514,7 +511,7 @@ def _create_resources_from_paths(self, root, paths): ) if not newpar: raise Exception( - f'ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path!r}' + f"ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path!r}" ) parent = newpar @@ -522,8 +519,8 @@ def _create_resources_from_paths(self, root, paths): if TRACE: logger_debug( - f' Codebase._create_resources_from_paths:', - f'created newpar: {newpar!r}', + f" Codebase._create_resources_from_paths:", + f"created newpar: {newpar!r}", ) res = self._get_or_create_resource( @@ -533,8 +530,7 @@ def _create_resources_from_paths(self, root, paths): is_file=isfile(res_loc), ) if TRACE: - logger_debug( - 'Codebase._create_resources_from_paths: resource', res) + logger_debug("Codebase._create_resources_from_paths: resource", res) def _create_resources_from_root(self, root): # without paths we walks the root location top-down @@ -547,11 +543,11 @@ def _create_resources_from_root(self, root): def err(_error): """os.walk error handler""" self.errors.append( - f'ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}' + f"ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}" ) # Walk over the directory and build the resource tree - for (top, dirs, files) in depth_walk( + for top, dirs, files in depth_walk( root_location=root.location, max_depth=self.max_depth, error_handler=err, @@ -586,7 +582,7 @@ def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored) is_file=is_file, ) if TRACE: - logger_debug('Codebase.create_resources:', res) + logger_debug("Codebase.create_resources:", res) yield res def _create_root_resource(self): @@ -595,8 +591,7 @@ def _create_root_resource(self): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError( - 'Root resource already exists and cannot be recreated') + raise TypeError("Root resource already exists and cannot be recreated") location = self.location name = file_name(location) @@ -605,8 +600,7 @@ def _create_root_resource(self): path = Resource.build_path(root_location=location, location=location) if TRACE: - logger_debug( - f' Codebase._create_root_resource: {path} is_file: {self.is_file}') + logger_debug(f" Codebase._create_root_resource: {path} is_file: {self.is_file}") logger_debug() root = self.resource_class( @@ -636,7 +630,7 @@ def _get_or_create_resource( """ if not parent: raise TypeError( - f'Cannot create resource without parent: name: {name!r}, path: {path!r}' + f"Cannot create resource without parent: name: {name!r}, path: {path!r}" ) # If the codebase is virtual, we provide the path @@ -647,13 +641,11 @@ def _get_or_create_resource( existing = self.get_resource(path) if existing: if TRACE: - logger_debug( - ' Codebase._get_or_create_resource: path already exists:', path) + logger_debug(" Codebase._get_or_create_resource: path already exists:", path) return existing if self._use_disk_cache_for_resource(): - cache_location = self._get_resource_cache_location( - path=path, create_dirs=True) + cache_location = self._get_resource_cache_location(path=path, create_dirs=True) else: cache_location = None @@ -666,8 +658,8 @@ def _get_or_create_resource( if TRACE: logger_debug( - f' Codebase._get_or_create_resource: with path: {path}\n' - f' name={name}, is_file={is_file}' + f" Codebase._get_or_create_resource: with path: {path}\n" + f" name={name}, is_file={is_file}" ) child = self.resource_class( @@ -698,7 +690,7 @@ def get_files_count(self): """ Return the final files counts for the codebase. """ - return self.counters.get('final:files_count', 0) + return self.counters.get("final:files_count", 0) def add_files_count_to_current_header(self): """ @@ -707,7 +699,7 @@ def add_files_count_to_current_header(self): """ files_count = self.get_files_count() current_header = self.get_or_create_current_header() - current_header.extra_data['files_count'] = files_count + current_header.extra_data["files_count"] = files_count return files_count def get_headers(self): @@ -743,10 +735,10 @@ def _use_disk_cache_for_resource(self): if TRACE: logger_debug( - f' Codebase._use_disk_cache_for_resource mode: {use_disk_cache} ' - f'on_disk: {self.all_on_disk} ' - f'in_mem: {self.all_in_memory} ' - f'max_in_mem: {self.max_in_memory}' + f" Codebase._use_disk_cache_for_resource mode: {use_disk_cache} " + f"on_disk: {self.all_on_disk} " + f"in_mem: {self.all_in_memory} " + f"max_in_mem: {self.max_in_memory}" ) return use_disk_cache @@ -763,8 +755,7 @@ def _exists_on_disk(self, path): """ path = clean_path(path) if not self._exists_in_memory(path): - cache_location = self._get_resource_cache_location( - path, create_dirs=False) + cache_location = self._get_resource_cache_location(path, create_dirs=False) if cache_location: return exists(cache_location) @@ -775,16 +766,15 @@ def get_resource(self, path): The ``path`` must be relative to the root (and including the root name as its first segment). """ - assert isinstance( - path, str), f'Invalid path: {path!r} is not a string.' + assert isinstance(path, str), f"Invalid path: {path!r} is not a string." path = clean_path(path) if TRACE: - msg = [' Codebase.get_resource:', 'path:', path] + msg = [" Codebase.get_resource:", "path:", path] if not path or path not in self.resources_by_path: - msg.append('not in resources!') + msg.append("not in resources!") else: - msg.extend(['exists on disk:', self._exists_on_disk(path)]) - msg.extend(['exists in memo:', self._exists_in_memory(path)]) + msg.extend(["exists on disk:", self._exists_on_disk(path)]) + msg.extend(["exists in memo:", self._exists_in_memory(path)]) logger_debug(*msg) # we use Codebase.CACHED_RESOURCE as a semaphore for existing but only @@ -801,11 +791,10 @@ def get_resource(self, path): pass else: # this should never happen - raise Exception( - f'get_resource: Internal error when getting {path!r}') + raise Exception(f"get_resource: Internal error when getting {path!r}") if TRACE: - logger_debug(' Resource:', res) + logger_debug(" Resource:", res) return res def save_resource(self, resource): @@ -818,7 +807,7 @@ def save_resource(self, resource): path = clean_path(resource.path) if TRACE: - logger_debug(' Codebase.save_resource:', resource) + logger_debug(" Codebase.save_resource:", resource) if resource.is_root: self.root = resource @@ -839,13 +828,12 @@ def _dump_resource(self, resource): if not cache_location: raise TypeError( - 'Resource cannot be dumped to disk and is used only' f'in memory: {resource}' + "Resource cannot be dumped to disk and is used only" f"in memory: {resource}" ) # TODO: consider messagepack or protobuf for compact/faster processing? - with open(cache_location, 'w') as cached: - cached.write(json.dumps( - resource.serialize(), check_circular=False)) + with open(cache_location, "w") as cached: + cached.write(json.dumps(resource.serialize(), check_circular=False)) # TODO: consider adding a small LRU cache in front of this for perf? def _load_resource(self, path): @@ -853,35 +841,32 @@ def _load_resource(self, path): Return a Resource with ``path`` loaded from the disk cache. """ path = clean_path(path) - cache_location = self._get_resource_cache_location( - path, create_dirs=False) + cache_location = self._get_resource_cache_location(path, create_dirs=False) if TRACE: logger_debug( - ' Codebase._load_resource: exists:', + " Codebase._load_resource: exists:", exists(cache_location), - 'cache_location:', + "cache_location:", cache_location, ) if not exists(cache_location): - raise ResourceNotInCache( - f'Failed to load Resource: {path} from {cache_location!r}') + raise ResourceNotInCache(f"Failed to load Resource: {path} from {cache_location!r}") # TODO: consider messagepack or protobuf for compact/faster processing try: - with open(cache_location, 'rb') as cached: + with open(cache_location, "rb") as cached: # TODO: Use custom json encoder to encode JSON list as a tuple # TODO: Consider using simplejson data = json.load(cached) return self.resource_class(**data) except Exception as e: - with open(cache_location, 'rb') as cached: + with open(cache_location, "rb") as cached: cached_data = cached.read() msg = ( - f'ERROR: failed to load resource from cached location: {cache_location} ' - 'with content:\n\n' + - repr(cached_data) + '\n\n' + traceback.format_exc() + f"ERROR: failed to load resource from cached location: {cache_location} " + "with content:\n\n" + repr(cached_data) + "\n\n" + traceback.format_exc() ) raise Exception(msg) from e @@ -891,13 +876,12 @@ def _remove_resource(self, resource): Does not remove children. """ if resource.is_root: - raise TypeError( - f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError(f"Cannot remove the root resource from codebase: {resource!r}") # remove from in-memory cache. The disk cache is cleared on exit. self.resources_by_path.pop(resource.path, None) if TRACE: - logger_debug('Codebase._remove_resource:', resource) + logger_debug("Codebase._remove_resource:", resource) def remove_resource(self, resource): """ @@ -905,12 +889,11 @@ def remove_resource(self, resource): codebase. Return a set of removed Resource paths. """ if TRACE: - logger_debug('Codebase.remove_resource') - logger_debug(' resource', resource) + logger_debug("Codebase.remove_resource") + logger_debug(" resource", resource) if resource.is_root: - raise TypeError( - f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError(f"Cannot remove the root resource from codebase: {resource!r}") removed_paths = set() @@ -923,7 +906,7 @@ def remove_resource(self, resource): # remove resource from parent parent = resource.parent(self) if TRACE: - logger_debug(' parent', parent) + logger_debug(" parent", parent) parent.children_names.remove(resource.name) parent.save(self) @@ -1026,7 +1009,7 @@ def update_counts(self, skip_filtered=False): try: resource._compute_children_counts(self, skip_filtered) except Exception as e: - msg = f'ERROR: cannot compute children counts for: {resource.path}' + msg = f"ERROR: cannot compute children counts for: {resource.path}" raise Exception(msg) from e def clear(self): @@ -1178,11 +1161,11 @@ class Resource(object): @property def type(self): - return 'file' if self.is_file else 'directory' + return "file" if self.is_file else "directory" @type.setter def type(self, value): - if value == 'file': + if value == "file": self.is_file = True else: self.is_file = False @@ -1207,10 +1190,10 @@ def build_path(cls, root_location, location): assert loc.startswith(root_loc) # keep the root directory name by default - root_loc = posixpath_parent(root_loc).strip('/') - path = loc.replace(root_loc, '', 1).strip('/') + root_loc = posixpath_parent(root_loc).strip("/") + path = loc.replace(root_loc, "", 1).strip("/") if TRACE: - logger_debug('build_path:', root_loc, loc, path) + logger_debug("build_path:", root_loc, loc, path) return path def get_path(self, full_root=False, strip_root=False): @@ -1282,7 +1265,7 @@ def extracted_to(self, codebase): """ Return the path this Resource archive was extracted to or None. """ - extract_path = f'{self.path}-extract' + extract_path = f"{self.path}-extract" return codebase.get_resource(extract_path) def extracted_from(self, codebase): @@ -1290,8 +1273,8 @@ def extracted_from(self, codebase): Return the path to an archive this Resource was extracted from or None. """ path = self.path - if '-extract' in path: - archive_path, _, _ = self.path.rpartition('-extract') + if "-extract" in path: + archive_path, _, _ = self.path.rpartition("-extract") return codebase.get_resource(archive_path) @classmethod @@ -1407,10 +1390,11 @@ def children(self, codebase, names=()): child_path = partial(posixpath_join, self.path) get_child = codebase.get_resource - children = [get_child(path=child_path(name)) - for name in children_names] + children = [get_child(path=child_path(name)) for name in children_names] + + def _sorter(r): + return (r.has_children(), r.name.lower(), r.name) - def _sorter(r): return (r.has_children(), r.name.lower(), r.name) return sorted((c for c in children if c), key=_sorter) def has_parent(self): @@ -1515,35 +1499,34 @@ def to_dict( return res if with_info: - res['name'] = self.name - res['base_name'] = self.base_name - res['extension'] = self.extension - res['size'] = self.size + res["name"] = self.name + res["base_name"] = self.base_name + res["extension"] = self.extension + res["size"] = self.size # exclude by default all of the "standard", default Resource fields self_fields_filter = attr.filters.exclude(*attr.fields(Resource)) # this will catch every attribute that has been added dynamically, such # as scan-provided resource_attributes - other_data = attr.asdict( - self, filter=self_fields_filter, dict_factory=dict) + other_data = attr.asdict(self, filter=self_fields_filter, dict_factory=dict) # FIXME: make a deep copy of the data first!!!! # see https://github.com/aboutcode-org/scancode-toolkit/issues/1199 res.update(other_data) if with_timing: - res['scan_time'] = self.scan_time or 0 - res['scan_timings'] = self.scan_timings or dict() + res["scan_time"] = self.scan_time or 0 + res["scan_timings"] = self.scan_timings or dict() if with_info: - res['files_count'] = self.files_count - res['dirs_count'] = self.dirs_count - res['size_count'] = self.size_count + res["files_count"] = self.files_count + res["dirs_count"] = self.dirs_count + res["size_count"] = self.size_count - res['scan_errors'] = self.scan_errors + res["scan_errors"] = self.scan_errors if TRACE: - logger_debug('Resource.to_dict:', res) + logger_debug("Resource.to_dict:", res) return res def serialize(self): @@ -1554,11 +1537,11 @@ def serialize(self): """ # we save all fields, not just the one in .to_dict() serializable = attr.asdict(self) - serializable['name'] = self.name + serializable["name"] = self.name if self.location: - serializable['location'] = self.location + serializable["location"] = self.location if self.cache_location: - serializable['cache_location'] = self.cache_location + serializable["cache_location"] = self.cache_location return serializable @@ -1566,11 +1549,11 @@ def clean_path(path): """ Return a cleaned and normalized POSIX ``path``. """ - path = path or '' + path = path or "" # convert to posix and ensure we have no slash at both ends - path = posixpath_normpath(path.replace('\\', '/').strip('/')) - if path == '.': - path = '' + path = posixpath_normpath(path.replace("\\", "/").strip("/")) + if path == ".": + path = "" return path @@ -1593,11 +1576,11 @@ def strip_first_path_segment(path): '' """ path = clean_path(path) - if '/' in path: - _root, _, path = path.partition('/') + if "/" in path: + _root, _, path = path.partition("/") return path else: - return '' + return "" def get_codebase_cache_dir(temp_dir): @@ -1609,13 +1592,12 @@ def get_codebase_cache_dir(temp_dir): from commoncode.fileutils import get_temp_dir from commoncode.timeutils import time2tstamp - prefix = 'scancode-codebase-' + time2tstamp() + '-' + prefix = "scancode-codebase-" + time2tstamp() + "-" return get_temp_dir(base_dir=temp_dir, prefix=prefix) @attr.s(slots=True) class _CodebaseAttributes(object): - def to_dict(self): return attr.asdict(self, dict_factory=dict) @@ -1626,7 +1608,7 @@ def from_attributes(cls, attributes): ``attributes`` mapping of "attr" attributes. """ return attr.make_class( - name='CodebaseAttributes', + name="CodebaseAttributes", attrs=attributes or {}, slots=True, bases=(_CodebaseAttributes,), @@ -1660,11 +1642,10 @@ def build_attributes_defs(mapping, ignored_keys=()): class VirtualCodebase(Codebase): - __slots__ = ( # TRUE iff the loaded virtual codebase has file information - 'with_info', - 'has_single_resource', + "with_info", + "has_single_resource", ) def __init__( @@ -1685,7 +1666,7 @@ def __init__( `max_depth`, if passed, will be ignored as VirtualCodebase will be using the depth of the original scan. """ - logger_debug(f'VirtualCodebase: new from: {location!r}') + logger_debug(f"VirtualCodebase: new from: {location!r}") self._setup_essentials(temp_dir, max_in_memory) @@ -1706,7 +1687,6 @@ def _get_scan_data_helper(self, location): try: return json.loads(location) except: - location = abspath(normpath(expanduser(location))) with open(location) as f: scan_data = json.load(f) @@ -1736,23 +1716,22 @@ def _get_scan_data(self, location): combined_scan_data = dict(headers=[], files=[]) for idx, loc in enumerate(location, 1): scan_data = self._get_scan_data_helper(loc) - headers = scan_data.get('headers') + headers = scan_data.get("headers") if headers: - combined_scan_data['headers'].extend(headers) - files = scan_data.get('files') + combined_scan_data["headers"].extend(headers) + files = scan_data.get("files") if files: for f in files: - f['path'] = posixpath_join( - f'codebase-{idx}', clean_path(f['path'])) - combined_scan_data['files'].extend(files) + f["path"] = posixpath_join(f"codebase-{idx}", clean_path(f["path"])) + combined_scan_data["files"].extend(files) else: raise Exception( f'Input file is missing a "files" (aka. resources) section to load: {loc}' ) - combined_scan_data['headers'] = sorted( - combined_scan_data['headers'], - key=lambda x: x['start_timestamp'], + combined_scan_data["headers"] = sorted( + combined_scan_data["headers"], + key=lambda x: x["start_timestamp"], ) return combined_scan_data @@ -1790,7 +1769,7 @@ def _collect_codebase_attributes(self, scan_data, *args, **kwargs): all_attributes = ( build_attributes_defs( mapping=scan_data, - ignored_keys=('headers', 'files'), + ignored_keys=("headers", "files"), ) or {} ) @@ -1811,7 +1790,7 @@ def _build_resource_class(self, sample_resource_data, *args, **kwargs): standard_res_attributes = set(f.name for f in attr.fields(Resource)) # add these properties since they are fields but are serialized - properties = set(['type', 'base_name', 'extension']) + properties = set(["type", "base_name", "extension"]) standard_res_attributes.update(properties) # We collect attributes that are not in standard_res_attributes already @@ -1829,7 +1808,7 @@ def _build_resource_class(self, sample_resource_data, *args, **kwargs): # Create the Resource class with the desired attributes return attr.make_class( - name='ScannedResource', + name="ScannedResource", attrs=all_res_attributes or dict(), slots=True, bases=(Resource,), @@ -1847,7 +1826,7 @@ def _populate(self, scan_data): """ # Collect headers ########################################################## - headers = scan_data.get('headers') or [] + headers = scan_data.get("headers") or [] headers = [Header.from_dict(**hle) for hle in headers] self.headers = headers @@ -1856,8 +1835,7 @@ def _populate(self, scan_data): # Codebase attributes to use. Configured with scan_data and plugin # attributes if present. self.codebase_attributes = self._collect_codebase_attributes(scan_data) - cbac = _CodebaseAttributes.from_attributes( - attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) self.attributes = cbac() # now populate top level codebase attributes @@ -1869,14 +1847,14 @@ def _populate(self, scan_data): setattr(self.attributes, attr_name, value) ########################################################## - files_data = scan_data.get('files') + files_data = scan_data.get("files") if not files_data: raise Exception('Input has no "files" top-level scan results.') if len(files_data) == 1: # we will shortcut to populate the codebase with a single root resource self.has_single_resource = True - root_is_file = files_data[0].get('type') == 'file' + root_is_file = files_data[0].get("type") == "file" else: root_is_file = False @@ -1908,9 +1886,9 @@ def _populate(self, scan_data): for fdata in files_data: sample_resource_data_update(fdata) - segments = fdata['path'].split('/') + segments = fdata["path"].split("/") root_names_add(segments[0]) - fdata['path_segments'] = segments + fdata["path_segments"] = segments # Resource sub-class to use. Configured with all known scanned file # attributes and plugin attributes if present @@ -1921,13 +1899,13 @@ def _populate(self, scan_data): self.with_info = any( a in sample_resource_data for a in ( - 'name', - 'base_name', - 'extension', - 'size', - 'files_count', - 'dirs_count', - 'size_count', + "name", + "base_name", + "extension", + "size", + "files_count", + "dirs_count", + "size_count", ) ) @@ -1935,28 +1913,28 @@ def _populate(self, scan_data): # Create root resource first ########################################################## if not root_names: - raise Exception('Unable to find root for codebase.') + raise Exception("Unable to find root for codebase.") len_root_names = len(root_names) if len_root_names == 1: root_path = root_names.pop() needs_new_virtual_root = False elif len_root_names > 1 or multiple_inputs: - root_path = 'virtual_root' + root_path = "virtual_root" needs_new_virtual_root = True if needs_new_virtual_root: for fdata in files_data: - rpath = fdata['path'] - fdata['path'] = posixpath_join(root_path, rpath) - fdata['path_segments'].insert(0, root_path) + rpath = fdata["path"] + fdata["path"] = posixpath_join(root_path, rpath) + fdata["path_segments"].insert(0, root_path) root_data = self._create_empty_resource_data() if self.has_single_resource: # single resource with one or more segments rdata = files_data[0] - root_path = rdata['path'] + root_path = rdata["path"] rdata = remove_properties_and_basics(rdata) root_data.update(rdata) @@ -1973,14 +1951,14 @@ def _populate(self, scan_data): setattr(root, name, value) if TRACE: - logger_debug('VirtualCodebase.populate: root:', root) + logger_debug("VirtualCodebase.populate: root:", root) # TODO: report error if filtering the root with a paths? self.save_resource(root) if self.has_single_resource: if TRACE: - logger_debug('VirtualCodebase.populate: with single resource.') + logger_debug("VirtualCodebase.populate: with single resource.") return all_paths = None @@ -1994,21 +1972,21 @@ def _populate(self, scan_data): # Note that we do not know the ordering there. # Therefore we sort in place by path segments - files_data.sort(key=itemgetter('path_segments')) + files_data.sort(key=itemgetter("path_segments")) # We create directories that exist in the scan or create these that # exist only in paths duplicated_paths = set() last_path = None for fdata in files_data: - path = fdata.get('path') + path = fdata.get("path") # skip the ones we did not request if all_paths and path not in all_paths: continue # these are no longer needed - path_segments = fdata.pop('path_segments') + path_segments = fdata.pop("path_segments") if not last_path: last_path = path @@ -2017,11 +1995,11 @@ def _populate(self, scan_data): else: last_path = path - name = fdata.get('name', None) or None + name = fdata.get("name", None) or None if not name: name = file_name(path) - is_file = fdata.get('type', 'file') == 'file' + is_file = fdata.get("type", "file") == "file" parent = self._get_parent_directory(path_segments=path_segments) resource = self._get_or_create_resource( @@ -2040,8 +2018,8 @@ def _populate(self, scan_data): if duplicated_paths: raise Exception( - 'Illegal combination of VirtualCode multiple inputs: ' - f'duplicated paths: {list(duplicated_paths)}', + "Illegal combination of VirtualCode multiple inputs: " + f"duplicated paths: {list(duplicated_paths)}", ) def _get_parent_directory(self, path_segments): @@ -2077,14 +2055,12 @@ def _create_root_resource(self, name, path, is_file): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError( - 'Root resource already exists and cannot be recreated') + raise TypeError("Root resource already exists and cannot be recreated") path = clean_path(path) if TRACE: - logger_debug( - f' VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}') + logger_debug(f" VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}") root = self.resource_class( name=name, @@ -2102,8 +2078,7 @@ def _create_root_resource(self, name, path, is_file): return root -KNOW_PROPS = set(['type', 'base_name', 'extension', - 'path', 'name', 'path_segments']) +KNOW_PROPS = set(["type", "base_name", "extension", "path", "name", "path_segments"]) def remove_properties_and_basics(resource_data): @@ -2128,10 +2103,10 @@ def get_ancestor_paths(path, include_self=False): >>> assert results == [], results """ assert path - segments = path.split('/') + segments = path.split("/") if not include_self: segments = segments[:-1] subpath = [] for segment in segments: subpath.append(segment) - yield '/'.join(subpath) + yield "/".join(subpath) diff --git a/src/commoncode/system.py b/src/commoncode/system.py index a2a43d2..0566a6c 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -16,22 +16,22 @@ def os_arch(): """ Return a tuple for the current the OS and architecture. """ - if sys.maxsize > 2 ** 32: - arch = '64' + if sys.maxsize > 2**32: + arch = "64" else: - arch = '32' + arch = "32" sys_platform = str(sys.platform).lower() - if sys_platform.startswith('linux'): - os = 'linux' - elif 'win32' in sys_platform: - os = 'win' - elif 'darwin' in sys_platform: - os = 'mac' - elif 'freebsd' in sys_platform: - os = 'freebsd' + if sys_platform.startswith("linux"): + os = "linux" + elif "win32" in sys_platform: + os = "win" + elif "darwin" in sys_platform: + os = "mac" + elif "freebsd" in sys_platform: + os = "freebsd" else: - raise Exception('Unsupported OS/platform %r' % sys_platform) + raise Exception("Unsupported OS/platform %r" % sys_platform) return os, arch @@ -39,17 +39,17 @@ def os_arch(): # OS/Arch # current_os, current_arch = os_arch() -on_windows = current_os == 'win' -on_windows_32 = on_windows and current_arch == '32' -on_windows_64 = on_windows and current_arch == '64' -on_mac = current_os == 'mac' -on_linux = current_os == 'linux' -on_freebsd = current_os == 'freebsd' +on_windows = current_os == "win" +on_windows_32 = on_windows and current_arch == "32" +on_windows_64 = on_windows and current_arch == "64" +on_mac = current_os == "mac" +on_linux = current_os == "linux" +on_freebsd = current_os == "freebsd" on_posix = not on_windows and (on_mac or on_linux or on_freebsd) -current_os_arch = '%(current_os)s-%(current_arch)s' % locals() -noarch = 'noarch' -current_os_noarch = '%(current_os)s-%(noarch)s' % locals() +current_os_arch = "%(current_os)s-%(current_arch)s" % locals() +noarch = "noarch" +current_os_noarch = "%(current_os)s-%(noarch)s" % locals() del os_arch @@ -61,10 +61,11 @@ def is_on_macos_14_or_higher(): filesystem encodings. """ import platform + macos_ver = platform.mac_ver() macos_ver = macos_ver[0] - macos_ver = macos_ver.split('.') - return macos_ver > ['10', '14'] + macos_ver = macos_ver.split(".") + return macos_ver > ["10", "14"] on_macos_14_or_higher = is_on_macos_14_or_higher() @@ -77,7 +78,8 @@ def is_on_macos_arm64(): Return True if the current OS is macOS running on Apple Silicon. """ import platform - return on_mac and platform.machine() == 'arm64' + + return on_mac and platform.machine() == "arm64" on_macos_arm64 = is_on_macos_arm64() @@ -85,14 +87,15 @@ def is_on_macos_arm64(): del is_on_macos_arm64 -def is_on_ubuntu_22(os_release_location='/etc/os-release'): +def is_on_ubuntu_22(os_release_location="/etc/os-release"): """ Return True if the current OS is Ubuntu 22.XX. """ if not on_linux: return False os_release_info = parse_os_release(os_release_location) - return os_release_info['ID'] == 'ubuntu' and '22' in os_release_info['VERSION_ID'] + return os_release_info["ID"] == "ubuntu" and "22" in os_release_info["VERSION_ID"] + try: on_ubuntu_22 = is_on_ubuntu_22() @@ -126,11 +129,11 @@ def has_case_sensitive_fs(): # Shared library file extensions # if on_windows: - lib_ext = '.dll' + lib_ext = ".dll" if on_mac: - lib_ext = '.dylib' + lib_ext = ".dylib" if on_linux or on_freebsd: - lib_ext = '.so' + lib_ext = ".so" # # Python versions @@ -157,5 +160,6 @@ def has_case_sensitive_fs(): # The system does not display the Windows Error Reporting dialog. if on_windows: import ctypes + # 3 is SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX ctypes.windll.kernel32.SetErrorMode(3) # @UndefinedVariable diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index 3fe6144..b24d4d1 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -12,15 +12,15 @@ import shutil import stat import sys -from os import path from collections import defaultdict from itertools import chain +from os import path from unittest import TestCase as TestCaseClass import saneyaml -from commoncode import fileutils from commoncode import filetype +from commoncode import fileutils from commoncode.archive import extract_tar from commoncode.archive import extract_tar_raw from commoncode.archive import extract_tar_uni @@ -42,13 +42,9 @@ def to_os_native_path(path): """ Normalize a path to use the native OS path separator. """ - OS_PATH_SEP = '\\' if on_windows else '/' + OS_PATH_SEP = "\\" if on_windows else "/" - return ( - path.replace('/', OS_PATH_SEP) - .replace(u'\\', OS_PATH_SEP) - .rstrip(OS_PATH_SEP) - ) + return path.replace("/", OS_PATH_SEP).replace("\\", OS_PATH_SEP).rstrip(OS_PATH_SEP) def get_test_loc( @@ -64,6 +60,7 @@ def get_test_loc( """ if debug: import inspect + caller = inspect.stack()[1][3] print('\nget_test_loc,%(caller)s,"%(test_path)s","%(test_data_dir)s"' % locals()) @@ -71,15 +68,18 @@ def get_test_loc( assert test_data_dir if not path.exists(test_data_dir): - raise IOError("[Errno 2] No such directory: test_data_dir not found:" - " '%(test_data_dir)s'" % locals()) + raise IOError( + "[Errno 2] No such directory: test_data_dir not found:" + " '%(test_data_dir)s'" % locals() + ) tpath = to_os_native_path(test_path) test_loc = path.abspath(path.join(test_data_dir, tpath)) if must_exist and not path.exists(test_loc): - raise IOError("[Errno 2] No such file or directory: " - "test_path not found: '%(test_loc)s'" % locals()) + raise IOError( + "[Errno 2] No such file or directory: " "test_path not found: '%(test_loc)s'" % locals() + ) return test_loc @@ -90,6 +90,7 @@ class FileDrivenTesting(object): temporary test resources and doing file-based assertions. This can be used as a standalone object if needed. """ + test_data_dir = None def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): @@ -104,6 +105,7 @@ def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): test_data_dir = self.test_data_dir if debug: import inspect + caller = inspect.stack()[1][3] print('\nself.get_test_loc,%(caller)s,"%(test_path)s"' % locals()) @@ -129,16 +131,16 @@ def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): test_loc = target_dir return test_loc - def get_temp_file(self, extension=None, dir_name='td', file_name='tf'): + def get_temp_file(self, extension=None, dir_name="td", file_name="tf"): """ Return a unique new temporary file location to a non-existing temporary file that can safely be created without a risk of name collision. """ if extension is None: - extension = '.txt' + extension = ".txt" - if extension and not extension.startswith('.'): - extension = '.' + extension + if extension and not extension.startswith("."): + extension = "." + extension file_name = file_name + extension temp_dir = self.get_temp_dir(dir_name) @@ -156,19 +158,19 @@ def get_temp_dir(self, sub_dir_path=None): global test_run_temp_dir if not test_run_temp_dir: import tempfile + test_tmp_root_dir = tempfile.gettempdir() # now we add a space in the path for testing path with spaces test_run_temp_dir = fileutils.get_temp_dir( - base_dir=test_tmp_root_dir, prefix='scancode-tk-tests -') + base_dir=test_tmp_root_dir, prefix="scancode-tk-tests -" + ) - test_run_temp_subdir = fileutils.get_temp_dir( - base_dir=test_run_temp_dir, prefix='') + test_run_temp_subdir = fileutils.get_temp_dir(base_dir=test_run_temp_dir, prefix="") if sub_dir_path: # create a sub directory hierarchy if requested sub_dir_path = to_os_native_path(sub_dir_path) - test_run_temp_subdir = path.join( - test_run_temp_subdir, sub_dir_path) + test_run_temp_subdir = path.join(test_run_temp_subdir, sub_dir_path) fileutils.create_dir(test_run_temp_subdir) return test_run_temp_subdir @@ -176,7 +178,7 @@ def remove_vcs(self, test_dir): """ Remove some version control directories and some temp editor files. """ - vcses = ('CVS', '.svn', '.git', '.hg') + vcses = ("CVS", ".svn", ".git", ".hg") for root, dirs, files in os.walk(test_dir): for vcs_dir in vcses: if vcs_dir in dirs: @@ -187,8 +189,9 @@ def remove_vcs(self, test_dir): shutil.rmtree(path.join(root, vcs_dir), False) # editors temp file leftovers - tilde_files = [path.join(root, file_loc) - for file_loc in files if file_loc.endswith('~')] + tilde_files = [ + path.join(root, file_loc) for file_loc in files if file_loc.endswith("~") + ] for tf in tilde_files: os.remove(tf) @@ -199,7 +202,7 @@ def __extract(self, test_path, extract_func=None, verbatim=False): archive file has been extracted using extract_func. If `verbatim` is True preserve the permissions. """ - assert test_path and test_path != '' + assert test_path and test_path != "" test_path = to_os_native_path(test_path) target_path = path.basename(test_path) target_dir = self.get_temp_dir(target_path) @@ -238,8 +241,7 @@ def phase3(self): Find out differences between common files. Ensure we are using content comparison, not os.stat-only. """ - comp = filecmp.cmpfiles(self.left, self.right, - self.common_files, shallow=False) + comp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) self.same_files, self.diff_files, self.funny_files = comp @@ -249,13 +251,11 @@ def is_same(dir1, dir2): Return False if they differ, True is they are the same. """ compared = dircmp(dir1, dir2) - if (compared.left_only or compared.right_only or compared.diff_files - or compared.funny_files): + if compared.left_only or compared.right_only or compared.diff_files or compared.funny_files: return False for subdir in compared.common_dirs: - if not is_same(path.join(dir1, subdir), - path.join(dir2, subdir)): + if not is_same(path.join(dir1, subdir), path.join(dir2, subdir)): return False return True @@ -265,14 +265,14 @@ def file_cmp(file1, file2, ignore_line_endings=False): Compare two files content. Return False if they differ, True is they are the same. """ - with open(file1, 'rb') as f1: + with open(file1, "rb") as f1: f1c = f1.read() if ignore_line_endings: - f1c = b'\n'.join(f1c.splitlines(False)) - with open(file2, 'rb') as f2: + f1c = b"\n".join(f1c.splitlines(False)) + with open(file2, "rb") as f2: f2c = f2.read() if ignore_line_endings: - f2c = b'\n'.join(f2c.splitlines(False)) + f2c = b"\n".join(f2c.splitlines(False)) assert f2c == f1c @@ -307,7 +307,7 @@ def make_non_executable(location): os.chmod(location, current_stat & ~stat.S_IEXEC) -def get_test_file_pairs(test_dir): +def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): """ Yield tuples of (data_file, test_file) from a test data `test_dir` directory. Raise exception for orphaned/dangling files. @@ -325,16 +325,16 @@ def get_test_file_pairs(test_dir): for top, _, files in os.walk(test_dir): for tfile in files: - if tfile.endswith('~'): + if tfile.endswith("~"): continue file_path = path.abspath(path.join(top, tfile)) - if tfile.endswith('.yml'): + if tfile.endswith(".yml"): data_file_path = file_path - test_file_path = file_path.replace('.yml', '') + test_file_path = file_path.replace(".yml", "") else: test_file_path = file_path - data_file_path = test_file_path + '.yml' + data_file_path = test_file_path + ".yml" if not path.exists(test_file_path): dangling_test_files.add(test_file_path) @@ -349,11 +349,13 @@ def get_test_file_pairs(test_dir): # ensure that we haev no dangling files if dangling_test_files or dangling_data_files: - msg = ['Dangling missing test files without a YAML data file:'] + \ - sorted(dangling_test_files) - msg += ['Dangling missing YAML data files without a test file'] + \ - sorted(dangling_data_files) - msg = '\n'.join(msg) + msg = ["Dangling missing test files without a YAML data file:"] + sorted( + dangling_test_files + ) + msg += ["Dangling missing YAML data files without a test file"] + sorted( + dangling_data_files + ) + msg = "\n".join(msg) print(msg) raise Exception(msg) @@ -361,28 +363,29 @@ def get_test_file_pairs(test_dir): diff = set(data_files.keys()).symmetric_difference(set(test_files.keys())) if diff: msg = [ - 'Orphaned copyright test file(s) found: ' - 'test file without its YAML test data file ' - 'or YAML test data file without its test file.'] + sorted(diff) - msg = '\n'.join(msg) + "Orphaned copyright test file(s) found: " + "test file without its YAML test data file " + "or YAML test data file without its test file." + ] + sorted(diff) + msg = "\n".join(msg) print(msg) raise Exception(msg) # ensure that test file paths are unique when you ignore case # we use the file names as test method names (and we have Windows that's # case insensitive - dupes = list(chain.from_iterable( - paths for paths in paths_ignoring_case.values() if len(paths) != 1)) + dupes = list( + chain.from_iterable(paths for paths in paths_ignoring_case.values() if len(paths) != 1) + ) if dupes: - msg = [ - 'Non unique test/data file(s) found when ignoring case!'] + sorted(dupes) + msg = ["Non unique test/data file(s) found when ignoring case!"] + sorted(dupes) - msg = '\n'.join(msg) + msg = "\n".join(msg) print(msg) raise Exception(msg) for test_file in test_files: - yield test_file + '.yml', test_file + yield test_file + ".yml", test_file def check_against_expected_json_file(results, expected_file, regen=False): @@ -394,8 +397,8 @@ def check_against_expected_json_file(results, expected_file, regen=False): This is convenient for updating tests expectations. But use with caution. """ if regen: - with open(expected_file, 'w') as reg: - json.dump(results, reg, indent=2, separators=(',', ': ')) + with open(expected_file, "w") as reg: + json.dump(results, reg, indent=2, separators=(",", ": ")) expected = results else: with open(expected_file) as exp: diff --git a/src/commoncode/text.py b/src/commoncode/text.py index b2f4ea1..ab56b0a 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -11,8 +11,8 @@ import re import unicodedata -from text_unidecode import unidecode from bs4.dammit import UnicodeDammit +from text_unidecode import unidecode """ A text processing module providing functions to process and prepare text @@ -49,24 +49,24 @@ def foldcase(text): def nopunc(): - return re.compile(r'[\W_]', re.MULTILINE | re.UNICODE) + return re.compile(r"[\W_]", re.MULTILINE | re.UNICODE) def nopunctuation(text): - u""" + """ Replaces any non alphanum symbol (i.e. punctuation) in text with space. Preserve the characters offsets by replacing punctuation with spaces. Warning: this also drops line endings. """ if not isinstance(text, str): text = as_unicode(text) - return re.sub(nopunc(), ' ', text) + return re.sub(nopunc(), " ", text) -CR = '\r' -LF = '\n' +CR = "\r" +LF = "\n" CRLF = CR + LF -CRLF_NO_CR = ' ' + LF +CRLF_NO_CR = " " + LF def unixlinesep(text, preserve=False): @@ -85,7 +85,7 @@ def nolinesep(text): """ if not isinstance(text, str): text = as_unicode(text) - return text.replace(CR, ' ').replace(LF, ' ') + return text.replace(CR, " ").replace(LF, " ") def toascii(s, translit=False): @@ -110,11 +110,11 @@ def toascii(s, translit=False): if translit: converted = unidecode(s) else: - converted = unicodedata.normalize('NFKD', s) + converted = unicodedata.normalize("NFKD", s) - converted = converted.replace('[?]', '_') - converted = converted.encode('ascii', 'ignore') - return converted.decode('ascii') + converted = converted.replace("[?]", "_") + converted = converted.encode("ascii", "ignore") + return converted.decode("ascii") def python_safe_name(s): @@ -126,9 +126,9 @@ def python_safe_name(s): s = toascii(s) s = foldcase(s) s = nopunctuation(s) - s = s.replace(' ', '_') - s = '_'.join(s.split()) - s = s.strip('_') + s = s.replace(" ", "_") + s = "_".join(s.split()) + s = s.strip("_") return s @@ -138,9 +138,9 @@ def as_unicode(s): """ if isinstance(s, str): return s - if s == b'': - return u'' + if s == b"": + return "" if not s: return s - assert isinstance(s, bytes), 's must be bytes but is: {}'.format(s) + assert isinstance(s, bytes), "s must be bytes but is: {}".format(s) return UnicodeDammit(s).markup diff --git a/src/commoncode/timeutils.py b/src/commoncode/timeutils.py index 0bcb011..bab8dad 100644 --- a/src/commoncode/timeutils.py +++ b/src/commoncode/timeutils.py @@ -25,7 +25,7 @@ def utcoffset(self, dt): # NOQA return None def tzname(self, dt): # NOQA - return 'UTC' + return "UTC" def dst(self, dt): # NOQA return None @@ -61,7 +61,7 @@ def time2tstamp(dt=None, path_safe=True): datim = dt or datetime.utcnow() iso = datim.isoformat() if path_safe: - iso = iso.replace(':', '').replace('/', '_') + iso = iso.replace(":", "").replace("/", "_") return iso @@ -70,11 +70,11 @@ def tstamp2time(stamp): Convert a UTC timestamp to a datetime object. """ # handle both basic and extended formats - tformat = '%Y-%m-%dT%H%M%S' if stamp[4] == '-' else '%Y%m%dT%H%M%S' + tformat = "%Y-%m-%dT%H%M%S" if stamp[4] == "-" else "%Y%m%dT%H%M%S" # normalize - dt_ms = stamp.strip().replace('Z', '').replace(':', '') + dt_ms = stamp.strip().replace("Z", "").replace(":", "") - dt_ms = dt_ms.split('.') + dt_ms = dt_ms.split(".") isodatim = dt_ms[0] datim = datetime.strptime(isodatim, tformat) # all stamps must be UTC diff --git a/src/commoncode/urn.py b/src/commoncode/urn.py index 37d9fa9..32dce79 100644 --- a/src/commoncode/urn.py +++ b/src/commoncode/urn.py @@ -75,21 +75,21 @@ class URNValidationError(Exception): # Describes the URN schema for each object type URN_SCHEMAS = { - 'license': { - 'object': 'license', - 'fields': ['key'], + "license": { + "object": "license", + "fields": ["key"], }, - 'owner': { - 'object': 'owner', - 'fields': ['name'], + "owner": { + "object": "owner", + "fields": ["name"], }, - 'component': { - 'object': 'component', - 'fields': ['name', 'version'], + "component": { + "object": "component", + "fields": ["name", "version"], }, - 'product': { - 'object': 'product', - 'fields': ['name', 'version'], + "product": { + "object": "product", + "fields": ["name", "version"], }, } @@ -107,13 +107,13 @@ def encode(object_type, **fields): # case is not significant for the object type object_type = object_type.strip().lower() - urn_prefix = 'urn:dje:{0}:'.format(quote_plus(object_type)) + urn_prefix = "urn:dje:{0}:".format(quote_plus(object_type)) - object_fields = URN_SCHEMAS[object_type]['fields'] + object_fields = URN_SCHEMAS[object_type]["fields"] # leading and trailing white spaces are not significant # each URN part is encoded individually BEFORE assembling the URN encoded_fields = [quote_plus(fields[f].strip()) for f in object_fields] - encoded_fields = ':'.join(encoded_fields) + encoded_fields = ":".join(encoded_fields) return urn_prefix + encoded_fields @@ -122,23 +122,23 @@ def decode(urn): Decode a URN and return the object_type and a mapping of field/values. Raise URNValidationError on errors. """ - segments = [unquote_plus(p) for p in urn.split(':')] + segments = [unquote_plus(p) for p in urn.split(":")] - if not segments[0] == ('urn'): + if not segments[0] == ("urn"): raise URNValidationError("Invalid URN prefix. Expected 'urn'.") - if not segments[1] == ('dje'): + if not segments[1] == ("dje"): raise URNValidationError("Invalid URN namespace. Expected 'dje'.") # object type is always lowercase object_type = segments[2].lower() if object_type not in URN_SCHEMAS: - raise URNValidationError('Unsupported URN object type.') + raise URNValidationError("Unsupported URN object type.") fields = segments[3:] - schema_fields = URN_SCHEMAS[object_type]['fields'] + schema_fields = URN_SCHEMAS[object_type]["fields"] if len(schema_fields) != len(fields): - raise URNValidationError('Invalid number of fields in URN.') + raise URNValidationError("Invalid number of fields in URN.") decoded_fields = dict(zip(schema_fields, fields)) return object_type, decoded_fields diff --git a/src/commoncode/version.py b/src/commoncode/version.py index 83cb246..a99ac8d 100644 --- a/src/commoncode/version.py +++ b/src/commoncode/version.py @@ -12,29 +12,31 @@ def VERSION_PATTERNS_REGEX(): - return [re.compile(x) for x in [ - # Eclipse features - r'v\d+\.feature\_(\d+\.){1,3}\d+', - - # Common version patterns - r'(M?(v\d+(\-|\_))?\d+\.){1,3}\d+[A-Za-z0-9]*((\.|\-|_|~)' - r'(b|B|rc|r|v|RC|alpha|beta|BETA|M|m|pre|vm|G)?\d+((\-|\.)\d+)?)?' - r'((\.|\-)(((alpha|dev|beta|rc|FINAL|final|pre)(\-|\_)\d+[A-Za-z]?' - r'(\-RELEASE)?)|alpha|dev(\.\d+\.\d+)?' - r'|beta|BETA|final|FINAL|release|fixed|(cr\d(\_\d*)?)))?', + return [ + re.compile(x) + for x in [ + # Eclipse features + r"v\d+\.feature\_(\d+\.){1,3}\d+", + # Common version patterns + r"(M?(v\d+(\-|\_))?\d+\.){1,3}\d+[A-Za-z0-9]*((\.|\-|_|~)" + r"(b|B|rc|r|v|RC|alpha|beta|BETA|M|m|pre|vm|G)?\d+((\-|\.)\d+)?)?" + r"((\.|\-)(((alpha|dev|beta|rc|FINAL|final|pre)(\-|\_)\d+[A-Za-z]?" + r"(\-RELEASE)?)|alpha|dev(\.\d+\.\d+)?" + r"|beta|BETA|final|FINAL|release|fixed|(cr\d(\_\d*)?)))?", # - r'[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+', + r"[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+", # - r'(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?', + r"(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?", # - r'current|previous|latest|alpha|beta', + r"current|previous|latest|alpha|beta", # - r'\d{4}-\d{2}-\d{2}', + r"\d{4}-\d{2}-\d{2}", # - r'(\d(\-|\_)){1,2}\d', + r"(\d(\-|\_)){1,2}\d", # - r'\d{5,14}', - ]] + r"\d{5,14}", + ] + ] def hint(path): @@ -43,7 +45,7 @@ def hint(path): the version does not start with v. """ for pattern in VERSION_PATTERNS_REGEX(): - segments = path.split('/') + segments = path.split("/") # skip the first path segment unless there's only one segment first_segment = 1 if len(segments) > 1 else 0 interesting_segments = segments[first_segment:] @@ -53,8 +55,8 @@ def hint(path): if version: v = version.group(0) # prefix with v space - if not v.lower().startswith('v'): - v = f'v {v}' + if not v.lower().startswith("v"): + v = f"v {v}" return v @@ -117,7 +119,7 @@ def is_moslty_num(s): return False -NameVersion = namedtuple('NameVersion', 'name, version') +NameVersion = namedtuple("NameVersion", "name, version") def get_jar_nv(filename): diff --git a/tests/test_cliutils.py b/tests/test_cliutils.py index a1d88ce..725eac9 100644 --- a/tests/test_cliutils.py +++ b/tests/test_cliutils.py @@ -6,118 +6,129 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from commoncode.cliutils import PluggableCommandLineOption +import os + +import click +from click.testing import CliRunner + from commoncode.cliutils import GroupedHelpCommand +from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import fixed_width_file_name from commoncode.testcase import FileDrivenTesting -from click.testing import CliRunner -import os -import click click.disable_unicode_literals_warning = True class TestFixedWidthFilename(FileDrivenTesting): - def test_fixed_width_file_name_with_file_name_larger_than_max_length_is_shortened(self): - test = fixed_width_file_name('0123456789012345678901234.c', 25) - expected = '0123456789...5678901234.c' + test = fixed_width_file_name("0123456789012345678901234.c", 25) + expected = "0123456789...5678901234.c" assert test == expected def test_fixed_width_file_name_with_file_name_smaller_than_max_length_is_not_shortened(self): - file_name = '0123456789012345678901234.c' + file_name = "0123456789012345678901234.c" test = fixed_width_file_name(file_name, max_length=50) assert test == file_name def test_fixed_width_file_name_with_file_name_at_max_length_is_not_shortened(self): - test = fixed_width_file_name('01234567890123456789012.c', 25) - expected = '01234567890123456789012.c' + test = fixed_width_file_name("01234567890123456789012.c", 25) + expected = "01234567890123456789012.c" assert test == expected def test_fixed_width_file_name_with_file_name_smaller_than_max_length_not_shortened(self): - test = fixed_width_file_name('0123456789012345678901.c', 25) - expected = '0123456789012345678901.c' + test = fixed_width_file_name("0123456789012345678901.c", 25) + expected = "0123456789012345678901.c" assert test == expected def test_fixed_width_file_name_with_none_filename_return_empty_string(self): test = fixed_width_file_name(None, 25) - expected = '' + expected = "" assert test == expected def test_fixed_width_file_name_without_extension(self): - test = fixed_width_file_name('012345678901234567890123456', 25) - expected = '01234567890...67890123456' + test = fixed_width_file_name("012345678901234567890123456", 25) + expected = "01234567890...67890123456" assert test == expected def test_fixed_width_file_name_with_posix_path_without_shortening(self): test = fixed_width_file_name( - 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js', 25) - expected = 'drupal.js' + "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js", 25 + ) + expected = "drupal.js" assert test == expected def test_fixed_width_file_name_with_posix_path_with_shortening(self): test = fixed_width_file_name( - 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c', 25) - expected = '0123456789...4567890123.c' + "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c", + 25, + ) + expected = "0123456789...4567890123.c" assert test == expected def test_fixed_width_file_name_with_win_path_without_shortening(self): test = fixed_width_file_name( - 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js', 25) - expected = 'drupal.js' + "C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js", 25 + ) + expected = "drupal.js" assert test == expected def test_fixed_width_file_name_with_win_path_with_shortening(self): test = fixed_width_file_name( - 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c', 25) - expected = '0123456789...4567890123.c' + "C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c", + 25, + ) + expected = "0123456789...4567890123.c" assert test == expected def test_fixed_width_file_name_with_very_small_file_name_and_long_extension(self): - test = fixed_width_file_name('abc.abcdef', 5) + test = fixed_width_file_name("abc.abcdef", 5) # FIXME: what is expected is TBD - expected = '' + expected = "" assert test == expected class TestGroupedHelpCommand(FileDrivenTesting): - - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_GroupedHelpCommand_help_group_and_sort_order_without_custom_class(self): - - @click.command(name='scan', cls=GroupedHelpCommand) - @click.option('--opt', is_flag=True, help='Help text for option') + @click.command(name="scan", cls=GroupedHelpCommand) + @click.option("--opt", is_flag=True, help="Help text for option") def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) + result = runner.invoke(scan, ["--help"]) from commoncode.cliutils import MISC_GROUP + assert MISC_GROUP in result.output - assert '--opt Help text for option' in result.output + assert "--opt Help text for option" in result.output def test_GroupedHelpCommand_with_help_group_and_sort_order_with_custom_class(self): - - @click.command(name='scan', cls=GroupedHelpCommand) - @click.option('--opt', is_flag=True, sort_order=10, - help='Help text for option', cls=PluggableCommandLineOption) + @click.command(name="scan", cls=GroupedHelpCommand) + @click.option( + "--opt", + is_flag=True, + sort_order=10, + help="Help text for option", + cls=PluggableCommandLineOption, + ) def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) + result = runner.invoke(scan, ["--help"]) from commoncode.cliutils import MISC_GROUP - assert MISC_GROUP + ':\n --opt Help text for option\n' in result.output + + assert MISC_GROUP + ":\n --opt Help text for option\n" in result.output def test_GroupedHelpCommand_help_with_group(self): from commoncode.cliutils import CORE_GROUP - @click.command(name='scan', cls=GroupedHelpCommand) + @click.command(name="scan", cls=GroupedHelpCommand) @click.option( - '--opt', + "--opt", is_flag=True, - help='Help text for option', + help="Help text for option", help_group=CORE_GROUP, cls=PluggableCommandLineOption, ) @@ -125,5 +136,5 @@ def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) - assert CORE_GROUP + ':\n --opt Help text for option\n' in result.output + result = runner.invoke(scan, ["--help"]) + assert CORE_GROUP + ":\n --opt Help text for option\n" in result.output diff --git a/tests/test_cliutils_progressbar.py b/tests/test_cliutils_progressbar.py index 1ee8dca..2683e9c 100644 --- a/tests/test_cliutils_progressbar.py +++ b/tests/test_cliutils_progressbar.py @@ -8,13 +8,12 @@ import os -from commoncode.testcase import FileDrivenTesting from commoncode.cliutils import progressmanager +from commoncode.testcase import FileDrivenTesting class TestProgressBar(FileDrivenTesting): - - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_progressmanager_works(self): iterable = range(10) diff --git a/tests/test_codec.py b/tests/test_codec.py index e852e70..1109acf 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -14,20 +14,19 @@ class TestCodec(TestCase): - def test_bin_to_num_basic(self): expected = 123 - result = bin_to_num(b'{') + result = bin_to_num(b"{") assert result == expected def test_bin_to_num_zero(self): expected = 0 - result = bin_to_num(b'\x00') + result = bin_to_num(b"\x00") assert result == expected def test_bin_to_num_large_number(self): expected = 432346237462348763 - result = bin_to_num(b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb') + result = bin_to_num(b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb") assert result == expected def test_bin_to_num_and_num_to_bin_is_idempotent(self): @@ -36,43 +35,43 @@ def test_bin_to_num_and_num_to_bin_is_idempotent(self): assert result == expected def test_num_to_bin_basic(self): - expected = b'{' + expected = b"{" result = num_to_bin(123) assert result == expected def test_num_to_bin_zero(self): - expected = b'\x00' + expected = b"\x00" result = num_to_bin(0) assert result == expected def test_num_to_bin_large_number(self): - expected = b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb' + expected = b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb" result = num_to_bin(432346237462348763) assert result == expected def test_num_to_bin_bin_to_num_is_idempotent(self): - expected = b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb' - result = num_to_bin(bin_to_num(b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb')) + expected = b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb" + result = num_to_bin(bin_to_num(b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb")) assert result == expected def test_urlsafe_b64encode_int_zero(self): - assert urlsafe_b64encode_int(0) == b'AA==' + assert urlsafe_b64encode_int(0) == b"AA==" def test_urlsafe_b64encode_int_basic(self): - assert urlsafe_b64encode_int(123123123123) == b'HKq1w7M=' + assert urlsafe_b64encode_int(123123123123) == b"HKq1w7M=" def test_urlsafe_b64encode_int_limit_8bits_255(self): - assert urlsafe_b64encode_int(255) == b'_w==' + assert urlsafe_b64encode_int(255) == b"_w==" def test_urlsafe_b64encode_int_limit_8bits_256(self): - assert urlsafe_b64encode_int(256) == b'AQA=' + assert urlsafe_b64encode_int(256) == b"AQA=" def test_urlsafe_b64encode_int_adds_no_padding_for_number_that_are_multiple_of_6_bits(self): - assert urlsafe_b64encode_int(0xFFFFFFFFFFFFFFFFFF) == b'____________' + assert urlsafe_b64encode_int(0xFFFFFFFFFFFFFFFFFF) == b"____________" assert len(urlsafe_b64encode_int(0xFFFFFFFFFFFF)) == 8 def test_urlsafe_b64encode_int_very_large_number(self): - b64 = (b'QAAAAAAgAAAAAQAACAAAAAAAAAAAAAAkAAIAAAAAAAAAAAAAAACAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAiAAAAAAAIAAAAAAAAAAAAAAEAACAAAAAAAA=') + b64 = b"QAAAAAAgAAAAAQAACAAAAAAAAAAAAAAkAAIAAAAAAAAAAAAAAACAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAiAAAAAAAIAAAAAAAAAAAAAAEAACAAAAAAAA=" expected = b64 num = 2678771517966886466622496485850735537232223496190189203248435106535830319026141316924949516664780383591425235756710588949364368366679435700855700642969357960349427980681242720502045830438444033569999428606714388704082526548154984676817460705606960919023941301616034362869262429593297635158449513824256 result = urlsafe_b64encode_int(num) diff --git a/tests/test_command.py b/tests/test_command.py index 1f54c73..173b6c6 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -9,150 +9,174 @@ import os import sys - from unittest.case import skipIf from commoncode import command -from commoncode.testcase import FileBasedTesting from commoncode.system import on_linux from commoncode.system import on_mac from commoncode.system import on_windows from commoncode.system import py36 +from commoncode.testcase import FileBasedTesting class TestCommand(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(py36, 'This fails on Python 3.6 https://bugs.python.org/issue26919') + @skipIf(py36, "This fails on Python 3.6 https://bugs.python.org/issue26919") def test_execute_can_handle_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these python = sys.executable rc, stdout, stderr = command.execute( - python, ['-c', 'print("non ascii: été just passed it !")'] + python, ["-c", 'print("non ascii: été just passed it !")'] ) - assert stderr == '' - assert stdout == 'non ascii: ete just passed it !' + assert stderr == "" + assert stdout == "non ascii: ete just passed it !" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") def test_execute_(self): python = sys.executable - rc, stdout, stderr = command.execute( - python, ['-c', 'print("foobar")'] - ) - assert stderr == '' - assert stdout == 'foobar' + rc, stdout, stderr = command.execute(python, ["-c", 'print("foobar")']) + assert stderr == "" + assert stdout == "foobar" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") def test_execute2(self): python = sys.executable - rc, stdout, stderr = command.execute2( - python, ['-c', 'print("foobar")'] - ) - assert stderr == '' - assert stdout == 'foobar' + rc, stdout, stderr = command.execute2(python, ["-c", 'print("foobar")']) + assert stderr == "" + assert stdout == "foobar" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") - @skipIf(not on_linux, 'Linux only') + @skipIf(not on_linux, "Linux only") def test_update_path_var_on_linux(self): - existing_path_var = '/usr/bin:/usr/local' + existing_path_var = "/usr/bin:/usr/local" - new_path = b'foo\xb1bar' + new_path = b"foo\xb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == 'foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'/bin/foo\udcb1bar' + new_path = "/bin/foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'/bin/foo\xb1bar' + new_path = b"/bin/foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'foo\xb1bar' + new_path = b"foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - @skipIf(not on_mac, 'Mac only') + @skipIf(not on_mac, "Mac only") def test_update_path_var_on_mac(self): - existing_path_var = '/usr/bin:/usr/local' + existing_path_var = "/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == 'foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'/bin/foo\xb1bar' + new_path = b"/bin/foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - @skipIf(not on_windows, 'Windows only') + @skipIf(not on_windows, "Windows only") def test_update_path_var_on_windows(self): - existing_path_var = u'c:\\windows;C:Program Files' + existing_path_var = "c:\\windows;C:Program Files" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == u'foo\udcb1bar;c:\\windows;C:Program Files' + assert updated_path == "foo\udcb1bar;c:\\windows;C:Program Files" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == u'foo\udcb1bar;c:\\windows;C:Program Files' + assert updated_path == "foo\udcb1bar;c:\\windows;C:Program Files" def test_searchable_paths(self): - d1 = self.get_temp_dir('foo') - d2 = self.get_temp_dir('bar') + d1 = self.get_temp_dir("foo") + d2 = self.get_temp_dir("bar") ps = os.pathsep - os.environ['FOO_SCANCODE_TEST1'] = f'{ps}{d1}{ps}{ps}{d2}{ps}' - os.environ['FOO_SCANCODE_TEST2'] = f'{ps}{d2}{ps}{ps}{d1}{ps}/NOTADIR' + os.environ["FOO_SCANCODE_TEST1"] = f"{ps}{d1}{ps}{ps}{d2}{ps}" + os.environ["FOO_SCANCODE_TEST2"] = f"{ps}{d2}{ps}{ps}{d1}{ps}/NOTADIR" - env_vars = 'FOO_SCANCODE_TEST1', 'FOO_SCANCODE_TEST2' + env_vars = "FOO_SCANCODE_TEST1", "FOO_SCANCODE_TEST2" expected = d1, d2, d2, d1 results = command.searchable_paths(env_vars=env_vars) if on_windows: for res, exp in zip(results, expected): - _, _, r = res.rpartition('\\') - _, _, e = exp.rpartition('\\') + _, _, r = res.rpartition("\\") + _, _, e = exp.rpartition("\\") assert r == e elif on_mac: # macOS somehow adds a /private to the paths in the CI as a side- # effect of calling "realpath" and likely resolving links - expected = f'/private{d1}', f'/private{d2}', f'/private{d2}', f'/private{d1}' + expected = f"/private{d1}", f"/private{d2}", f"/private{d2}", f"/private{d1}" assert expected == results else: assert expected == results def test_find_in_path(self): - d1 = self.get_temp_dir('foo') - d2 = self.get_temp_dir('bar') - filename = 'baz' + d1 = self.get_temp_dir("foo") + d2 = self.get_temp_dir("bar") + filename = "baz" assert None == command.find_in_path( - filename, searchable_paths=(d1, d2,)) + filename, + searchable_paths=( + d1, + d2, + ), + ) f2 = os.path.join(d2, filename) - with open(f2, 'w') as o: - o.write('some') - - assert f2 == command.find_in_path(filename, searchable_paths=(d1, d2,)) - assert f2 == command.find_in_path(filename, searchable_paths=(d2, d1,)) + with open(f2, "w") as o: + o.write("some") + + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d1, + d2, + ), + ) + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d2, + d1, + ), + ) f1 = os.path.join(d1, filename) - with open(f1, 'w') as o: - o.write('some') - - assert f1 == command.find_in_path(filename, searchable_paths=(d1, d2,)) - assert f2 == command.find_in_path(filename, searchable_paths=(d2, d1,)) + with open(f1, "w") as o: + o.write("some") + + assert f1 == command.find_in_path( + filename, + searchable_paths=( + d1, + d2, + ), + ) + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d2, + d1, + ), + ) diff --git a/tests/test_date.py b/tests/test_date.py index 4f30f27..05a2966 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -7,19 +7,16 @@ # import os - from datetime import datetime import commoncode.date - from commoncode import testcase class TestDate(testcase.FileBasedTesting): - def test_secs_from_epoch_can_handle_micro_and_nano_secs(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() # setting modified time to desired values os.utime(test_file, (1301420665.046481, 1301420665.046481)) # otherwise the issue does not happen (ie. on mac) @@ -29,19 +26,19 @@ def test_secs_from_epoch_can_handle_micro_and_nano_secs(self): def test_get_file_mtime_for_a_new_file(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() def as_yyyymmdd(s): return s[:10] - now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") result = commoncode.date.get_file_mtime(test_file) assert as_yyyymmdd(result) == as_yyyymmdd(now) def test_get_file_mtime_for_a_modified_file(self): test_file = self.get_temp_file() - open(test_file, 'w').close() - expected = u'1992-05-09 00:00:00' + open(test_file, "w").close() + expected = "1992-05-09 00:00:00" m_ts = (24 * 3600) * 134 + (24 * 3600 * 365) * 22 # setting modified time to expected values os.utime(test_file, (m_ts, m_ts)) @@ -49,8 +46,8 @@ def test_get_file_mtime_for_a_modified_file(self): def test_get_file_mtime_for_a_modified_file_2(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() # setting modified time to expected values - expected = u'2011-01-06 14:35:00' + expected = "2011-01-06 14:35:00" os.utime(test_file, (1294324500, 1294324500)) assert commoncode.date.get_file_mtime(test_file) == expected diff --git a/tests/test_distro.py b/tests/test_distro.py index c4b5d5a..48521c1 100644 --- a/tests/test_distro.py +++ b/tests/test_distro.py @@ -7,14 +7,14 @@ class TestDistro(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_parse_os_release(self): - test_dir = self.get_test_loc('distro/os-release') + test_dir = self.get_test_loc("distro/os-release") for test_file in resource_iter(test_dir, with_dirs=False): - if test_file.endswith('expected.json'): + if test_file.endswith("expected.json"): continue - expected = test_file + '-expected.json' + expected = test_file + "-expected.json" result = distro.parse_os_release(test_file) check_against_expected_json_file(result, expected, regen=False) diff --git a/tests/test_fileset.py b/tests/test_fileset.py index 79e44a8..2c6e5ef 100644 --- a/tests/test_fileset.py +++ b/tests/test_fileset.py @@ -13,82 +13,84 @@ class FilesetTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_load(self): - irf = self.get_test_loc('fileset/scancodeignore.lst') + irf = self.get_test_loc("fileset/scancodeignore.lst") result = fileset.load(irf) - assert result == ['/foo/*', '!/foobar/*', 'bar/*', '#comment'] + assert result == ["/foo/*", "!/foobar/*", "bar/*", "#comment"] def test_is_included_basic(self): - assert fileset.is_included('/common/src/', {}, {}) - assert fileset.is_included('/common/src/', None, None) + assert fileset.is_included("/common/src/", {}, {}) + assert fileset.is_included("/common/src/", None, None) assert not fileset.is_included(None, None, None) def test_is_included_in_fileset(self): - incs = {'/common/src/*': '.scanignore'} - excs = {'/common/src/*.so': '.scanignore'} + incs = {"/common/src/*": ".scanignore"} + excs = {"/common/src/*.so": ".scanignore"} assert not fileset.is_included(None, incs, excs) - assert not fileset.is_included('', incs, excs) - assert not fileset.is_included('/', incs, excs) - assert fileset.is_included('/common/src/', incs, excs) - assert not fileset.is_included('/common/bin/', incs, excs) + assert not fileset.is_included("", incs, excs) + assert not fileset.is_included("/", incs, excs) + assert fileset.is_included("/common/src/", incs, excs) + assert not fileset.is_included("/common/bin/", incs, excs) def test_is_included_in_fileset_2(self): - incs = {'src*': '.scanignore'} - excs = {'src/ab': '.scanignore'} + incs = {"src*": ".scanignore"} + excs = {"src/ab": ".scanignore"} assert not fileset.is_included(None, incs, excs) - assert not fileset.is_included('', incs, excs) - assert not fileset.is_included('/', incs, excs) - assert fileset.is_included('/common/src/', incs, excs) - assert not fileset.is_included('src/ab', incs, excs) - assert fileset.is_included('src/abbab', incs, excs) + assert not fileset.is_included("", incs, excs) + assert not fileset.is_included("/", incs, excs) + assert fileset.is_included("/common/src/", incs, excs) + assert not fileset.is_included("src/ab", incs, excs) + assert fileset.is_included("src/abbab", incs, excs) def test_is_included_is_included_exclusions(self): - incs = {'/src/*': '.scanignore'} - excs = {'/src/*.so': '.scanignore'} - assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"/src/*": ".scanignore"} + excs = {"/src/*.so": ".scanignore"} + assert not fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_is_included_exclusions_2(self): - incs = {'src': '.scanignore'} - excs = {'src/*.so': '.scanignore'} - assert fileset.is_included('/some/src/this/that', incs, excs) - assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"src": ".scanignore"} + excs = {"src/*.so": ".scanignore"} + assert fileset.is_included("/some/src/this/that", incs, excs) + assert not fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_empty_exclusions(self): - incs = {'/src/*': '.scanignore'} - excs = {'': '.scanignore'} - assert fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"/src/*": ".scanignore"} + excs = {"": ".scanignore"} + assert fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_sources(self): - incs = {'/home/elf/elf-0.5/*': '.scanignore'} - excs = {'/home/elf/elf-0.5/src/elf': '.scanignore', - '/home/elf/elf-0.5/src/elf.o': '.scanignore'} - assert not fileset.is_included('/home/elf/elf-0.5/src/elf', incs, excs) + incs = {"/home/elf/elf-0.5/*": ".scanignore"} + excs = { + "/home/elf/elf-0.5/src/elf": ".scanignore", + "/home/elf/elf-0.5/src/elf.o": ".scanignore", + } + assert not fileset.is_included("/home/elf/elf-0.5/src/elf", incs, excs) def test_is_included_dot_svn(self): - incs = {'*/.svn/*': '.scanignore'} + incs = {"*/.svn/*": ".scanignore"} excs = {} - assert fileset.is_included('home/common/tools/elf/.svn/', incs, excs) - assert fileset.is_included('home/common/tools/.svn/this', incs, excs) - assert not fileset.is_included('home/common/tools/this', incs, excs) + assert fileset.is_included("home/common/tools/elf/.svn/", incs, excs) + assert fileset.is_included("home/common/tools/.svn/this", incs, excs) + assert not fileset.is_included("home/common/tools/this", incs, excs) def test_is_included_dot_svn_with_excludes(self): - incs = {'*/.svn/*': '.scanignore'} - excs = {'*/.git/*': '.scanignore'} - assert fileset.is_included('home/common/tools/elf/.svn/', incs, excs) - assert fileset.is_included('home/common/tools/.svn/this', incs, excs) - assert not fileset.is_included('home/common/.git/this', incs, excs) + incs = {"*/.svn/*": ".scanignore"} + excs = {"*/.git/*": ".scanignore"} + assert fileset.is_included("home/common/tools/elf/.svn/", incs, excs) + assert fileset.is_included("home/common/tools/.svn/this", incs, excs) + assert not fileset.is_included("home/common/.git/this", incs, excs) def test_get_matches(self): - patterns = {'*/.svn/*': '.scanignore'} - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) - assert fileset.get_matches('home/common/tools/.svn/this', patterns) - assert not fileset.get_matches('home/common/.git/this', patterns) + patterns = {"*/.svn/*": ".scanignore"} + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) + assert fileset.get_matches("home/common/tools/.svn/this", patterns) + assert not fileset.get_matches("home/common/.git/this", patterns) def test_get_matches_accepts_a_list_or_tuple(self): - patterns = ['*/.svn/*'] - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) + patterns = ["*/.svn/*"] + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) - patterns = '*/.svn/*', - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) + patterns = ("*/.svn/*",) + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) diff --git a/tests/test_filetype.py b/tests/test_filetype.py index 378313a..5a8a14e 100644 --- a/tests/test_filetype.py +++ b/tests/test_filetype.py @@ -7,35 +7,34 @@ # import os -from os.path import join from os.path import exists +from os.path import join from unittest import skipIf +import commoncode.testcase from commoncode import filetype from commoncode import fileutils from commoncode.system import on_posix from commoncode.system import on_windows from commoncode.system import py3 - -import commoncode.testcase from commoncode.testcase import FileBasedTesting from commoncode.testcase import make_non_readable from commoncode.testcase import make_non_writable class TypeTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_get_size_on_file(self): - test_file = self.get_test_loc('filetype/size/Image1.eps') + test_file = self.get_test_loc("filetype/size/Image1.eps") assert filetype.get_size(test_file) == 12388 def test_get_size_on_directory(self): - test_dir = self.get_test_loc('filetype/size', copy=True) + test_dir = self.get_test_loc("filetype/size", copy=True) assert filetype.get_size(test_dir) == 12400 def test_get_type(self): - test_dir = self.extract_test_tar('filetype/types.tar', verbatim=True) + test_dir = self.extract_test_tar("filetype/types.tar", verbatim=True) results = [] for root, dirs, files in os.walk(test_dir): for d in dirs: @@ -44,27 +43,33 @@ def test_get_type(self): results.append((f, filetype.get_type(os.path.join(root, f)))) expected = [ - ('5-DIRTYPE', 'd'), - ('0-REGTYPE', 'f'), - ('0-REGTYPE-TEXT', 'f'), - ('0-REGTYPE-VEEEERY_LONG_NAME___________________________________' - '______________________________________________________________' - '____________________155', 'f'), - ('1-LNKTYPE', 'f'), - ('S-SPARSE', 'f'), - ('S-SPARSE-WITH-NULLS', 'f') + ("5-DIRTYPE", "d"), + ("0-REGTYPE", "f"), + ("0-REGTYPE-TEXT", "f"), + ( + "0-REGTYPE-VEEEERY_LONG_NAME___________________________________" + "______________________________________________________________" + "____________________155", + "f", + ), + ("1-LNKTYPE", "f"), + ("S-SPARSE", "f"), + ("S-SPARSE-WITH-NULLS", "f"), ] # symlinks and special files are not supported on win if on_posix: - expected += [('2-SYMTYPE', 'l'), ('6-FIFOTYPE', 's'), ] + expected += [ + ("2-SYMTYPE", "l"), + ("6-FIFOTYPE", "s"), + ] try: assert sorted(results) == sorted(expected) except Exception as e: if on_windows and py3: # On some Windows symlinkes are detected OK (Windows 10?) but not in Windows 7 - expected += [('2-SYMTYPE', 'l')] + expected += [("2-SYMTYPE", "l")] assert sorted(results) == sorted(expected) else: raise e @@ -75,8 +80,8 @@ def test_is_rwx_with_none(self): assert not filetype.is_executable(None) def test_is_readable_is_writeable_file(self): - base_dir = self.get_test_loc('filetype/readwrite', copy=True) - test_file = os.path.join(os.path.join(base_dir, 'sub'), 'file') + base_dir = self.get_test_loc("filetype/readwrite", copy=True) + test_file = os.path.join(os.path.join(base_dir, "sub"), "file") try: assert filetype.is_readable(test_file) @@ -92,8 +97,8 @@ def test_is_readable_is_writeable_file(self): fileutils.chmod(base_dir, fileutils.RW, recurse=True) def test_is_readable_is_writeable_dir(self): - base_dir = self.get_test_loc('filetype/readwrite', copy=True) - test_dir = os.path.join(base_dir, 'sub') + base_dir = self.get_test_loc("filetype/readwrite", copy=True) + test_dir = os.path.join(base_dir, "sub") try: assert filetype.is_readable(test_dir) @@ -117,11 +122,11 @@ def test_is_readable_is_writeable_dir(self): class CountTest(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def get_test_count_dir(self): - test_dir = self.get_test_loc('count/filecount', copy=True) - sub3 = join(test_dir, 'dir', 'sub3') + test_dir = self.get_test_loc("count/filecount", copy=True) + sub3 = join(test_dir, "dir", "sub3") if not exists(sub3): os.makedirs(sub3) return test_dir @@ -132,8 +137,8 @@ def test_get_file_count_with_empty_dir(self): def test_get_file_count_with_single_file(self): test_file = self.get_temp_file() - with open(test_file, 'w') as f: - f.write(u'') + with open(test_file, "w") as f: + f.write("") assert filetype.is_file(test_file) assert filetype.get_file_count(test_file) == 1 @@ -150,21 +155,21 @@ def test_get_file_size_and_count(self): def test_get_file_size(self): test_dir = self.get_test_count_dir() tests = ( - ('dir/a.txt', 2), - ('dir/b.txt', 2), - ('dir/c.txt', 2), - ('dir/sub1/a.txt', 2), - ('dir/sub1/b.txt', 2), - ('dir/sub1/c.txt', 2), - ('dir/sub1/subsub/a.txt', 2), - ('dir/sub1/subsub/b.txt', 2), - ('dir/sub1/subsub', 4), - ('dir/sub1', 10), - ('dir/sub2/a.txt', 2), - ('dir/sub2', 2), - ('dir/sub3', 0), - ('dir/', 18), - ('', 18), + ("dir/a.txt", 2), + ("dir/b.txt", 2), + ("dir/c.txt", 2), + ("dir/sub1/a.txt", 2), + ("dir/sub1/b.txt", 2), + ("dir/sub1/c.txt", 2), + ("dir/sub1/subsub/a.txt", 2), + ("dir/sub1/subsub/b.txt", 2), + ("dir/sub1/subsub", 4), + ("dir/sub1", 10), + ("dir/sub2/a.txt", 2), + ("dir/sub2", 2), + ("dir/sub3", 0), + ("dir/", 18), + ("", 18), ) for test_file, size in tests: result = filetype.get_size(os.path.join(test_dir, test_file)) @@ -173,21 +178,21 @@ def test_get_file_size(self): def test_get_file_count(self): test_dir = self.get_test_count_dir() tests = ( - ('dir/a.txt', 1), - ('dir/b.txt', 1), - ('dir/c.txt', 1), - ('dir/sub1/a.txt', 1), - ('dir/sub1/b.txt', 1), - ('dir/sub1/c.txt', 1), - ('dir/sub1/subsub/a.txt', 1), - ('dir/sub1/subsub/b.txt', 1), - ('dir/sub1/subsub', 2), - ('dir/sub1', 5), - ('dir/sub2/a.txt', 1), - ('dir/sub2', 1), - ('dir/sub3', 0), - ('dir/', 9), - ('', 9), + ("dir/a.txt", 1), + ("dir/b.txt", 1), + ("dir/c.txt", 1), + ("dir/sub1/a.txt", 1), + ("dir/sub1/b.txt", 1), + ("dir/sub1/c.txt", 1), + ("dir/sub1/subsub/a.txt", 1), + ("dir/sub1/subsub/b.txt", 1), + ("dir/sub1/subsub", 2), + ("dir/sub1", 5), + ("dir/sub2/a.txt", 1), + ("dir/sub2", 1), + ("dir/sub3", 0), + ("dir/", 9), + ("", 9), ) for test_file, count in tests: result = filetype.get_file_count(os.path.join(test_dir, test_file)) @@ -195,22 +200,22 @@ def test_get_file_count(self): class SymlinkTest(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_is_file(self): - test_file = self.get_test_loc('symlink/test', copy=True) + test_file = self.get_test_loc("symlink/test", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-link') + test_link = join(temp_dir, "test-link") os.symlink(test_file, test_link) assert filetype.is_file(test_link, follow_symlinks=True) assert not filetype.is_file(test_link, follow_symlinks=False) - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_is_dir(self): - test_dir = self.get_test_loc('symlink', copy=True) + test_dir = self.get_test_loc("symlink", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) assert filetype.is_dir(test_link, follow_symlinks=True) assert not filetype.is_dir(test_link, follow_symlinks=False) diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index 049c1c7..73cedc1 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -16,9 +16,9 @@ from commoncode import fileutils from commoncode.fileutils import as_posixpath from commoncode.system import on_linux -from commoncode.system import on_posix from commoncode.system import on_mac from commoncode.system import on_macos_14_or_higher +from commoncode.system import on_posix from commoncode.system import on_windows from commoncode.testcase import FileBasedTesting from commoncode.testcase import make_non_executable @@ -26,17 +26,18 @@ from commoncode.testcase import make_non_writable -@skip('Somehow permissions tests do not work OK yet on Python 3') +@skip("Somehow permissions tests do not work OK yet on Python 3") class TestPermissionsDeletions(FileBasedTesting): """ This is failing for now on Python 3 """ - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_delete_unwritable_directory_and_files(self): - base_dir = self.get_test_loc('fileutils/readwrite', copy=True) - test_dir = join(base_dir, 'sub') - test_file = join(test_dir, 'file') + base_dir = self.get_test_loc("fileutils/readwrite", copy=True) + test_dir = join(base_dir, "sub") + test_file = join(test_dir, "file") try: # note: there are no unread/writable dir on windows @@ -59,16 +60,17 @@ class TestPermissions(FileBasedTesting): Several assertions or test are skipped on non posix OSes. Windows handles permissions and special files differently. """ - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_chmod_on_non_existing_file_throws_no_exception(self): - fileutils.chmod('some non existing dir', fileutils.RWX) + fileutils.chmod("some non existing dir", fileutils.RWX) def test_chmod_read_write_recursively_on_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') - test_dir2 = join(test_dir, 'deep1', 'deep2') - parent = join(test_dir, 'deep1') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") + test_dir2 = join(test_dir, "deep1", "deep2") + parent = join(test_dir, "deep1") try: make_non_writable(test_file) @@ -94,10 +96,10 @@ def test_chmod_read_write_recursively_on_dir(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_non_recursively_on_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') - test_dir = join(test_dir, 'deep1', 'deep2') - parent = join(test_dir, 'deep1') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") + test_dir = join(test_dir, "deep1", "deep2") + parent = join(test_dir, "deep1") try: # setup @@ -124,8 +126,8 @@ def test_chmod_read_write_non_recursively_on_dir(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_file(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") try: make_non_writable(test_file) @@ -138,8 +140,8 @@ def test_chmod_read_write_file(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_exec_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") try: if on_posix: @@ -158,8 +160,8 @@ def test_chmod_read_write_exec_dir(self): def test_copyfile_does_not_keep_permissions(self): src_file = self.get_temp_file() dest = self.get_temp_dir() - with open(src_file, 'w') as f: - f.write(u'') + with open(src_file, "w") as f: + f.write("") try: make_non_readable(src_file) if on_posix: @@ -173,28 +175,28 @@ def test_copyfile_does_not_keep_permissions(self): fileutils.chmod(dest, fileutils.RW, recurse=True) def test_copytree_does_not_keep_non_writable_permissions(self): - src = self.get_test_loc('fileutils/exec', copy=True) + src = self.get_test_loc("fileutils/exec", copy=True) dst = self.get_temp_dir() try: - src_file = join(src, 'subtxt/a.txt') + src_file = join(src, "subtxt/a.txt") make_non_writable(src_file) assert not filetype.is_writable(src_file) - src_dir = join(src, 'subtxt') + src_dir = join(src, "subtxt") make_non_writable(src_dir) if on_posix: assert not filetype.is_writable(src_dir) # copy proper - dest_dir = join(dst, 'dest') + dest_dir = join(dst, "dest") fileutils.copytree(src, dest_dir) - dst_file = join(dest_dir, 'subtxt/a.txt') + dst_file = join(dest_dir, "subtxt/a.txt") assert os.path.exists(dst_file) assert filetype.is_writable(dst_file) - dest_dir2 = join(dest_dir, 'subtxt') + dest_dir2 = join(dest_dir, "subtxt") assert os.path.exists(dest_dir2) assert filetype.is_writable(dest_dir) finally: @@ -202,10 +204,10 @@ def test_copytree_does_not_keep_non_writable_permissions(self): fileutils.chmod(dst, fileutils.RW, recurse=True) def test_copytree_copies_unreadable_files(self): - src = self.get_test_loc('fileutils/exec', copy=True) + src = self.get_test_loc("fileutils/exec", copy=True) dst = self.get_temp_dir() - src_file1 = join(src, 'a.bat') - src_file2 = join(src, 'subtxt', 'a.txt') + src_file1 = join(src, "a.bat") + src_file2 = join(src, "subtxt", "a.txt") try: # make some unreadable source files @@ -218,14 +220,14 @@ def test_copytree_copies_unreadable_files(self): assert not filetype.is_readable(src_file2) # copy proper - dest_dir = join(dst, 'dest') + dest_dir = join(dst, "dest") fileutils.copytree(src, dest_dir) - dest_file1 = join(dest_dir, 'a.bat') + dest_file1 = join(dest_dir, "a.bat") assert os.path.exists(dest_file1) assert filetype.is_readable(dest_file1) - dest_file2 = join(dest_dir, 'subtxt', 'a.txt') + dest_file2 = join(dest_dir, "subtxt", "a.txt") assert os.path.exists(dest_file2) assert filetype.is_readable(dest_file2) @@ -235,21 +237,21 @@ def test_copytree_copies_unreadable_files(self): class TestFileUtils(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_windows, 'Windows handles special files differently.') + @skipIf(on_windows, "Windows handles special files differently.") def test_copytree_does_not_copy_fifo(self): # Windows does not support pipes - src = self.get_test_loc('fileutils/filetype', copy=True) + src = self.get_test_loc("fileutils/filetype", copy=True) dest = self.get_temp_dir() - src_file = join(src, 'myfifo') + src_file = join(src, "myfifo") os.mkfifo(src_file) # NOQA - dest_dir = join(dest, 'dest') + dest_dir = join(dest, "dest") fileutils.copytree(src, dest_dir) - assert not os.path.exists(join(dest_dir, 'myfifo')) + assert not os.path.exists(join(dest_dir, "myfifo")) def test_copyfile_keeps_modified_date(self): - test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True) + test_file = self.get_test_loc("fileutils/exec/subtxt/a.txt", copy=True) dest = self.get_temp_file() expected = 1289918700 os.utime(test_file, (expected, expected)) @@ -258,25 +260,25 @@ def test_copyfile_keeps_modified_date(self): assert result == expected def test_copyfile_can_copy_file_to_dir_keeping_full_file_name(self): - test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True) + test_file = self.get_test_loc("fileutils/exec/subtxt/a.txt", copy=True) dest = self.get_temp_dir() - expected = os.path.join(dest, 'a.txt') + expected = os.path.join(dest, "a.txt") fileutils.copyfile(test_file, dest) assert os.path.exists(expected) def test_resource_name(self): - assert fileutils.resource_name('/a/b/d/f/f') == 'f' - assert fileutils.resource_name('/a/b/d/f/f/') == 'f' - assert fileutils.resource_name('a/b/d/f/f/') == 'f' - assert fileutils.resource_name('/a/b/d/f/f.a') == 'f.a' - assert fileutils.resource_name('/a/b/d/f/f.a/') == 'f.a' - assert fileutils.resource_name('a/b/d/f/f.a') == 'f.a' - assert fileutils.resource_name('f.a') == 'f.a' - - @skipIf(on_windows, 'Windows FS encoding is ... different!') + assert fileutils.resource_name("/a/b/d/f/f") == "f" + assert fileutils.resource_name("/a/b/d/f/f/") == "f" + assert fileutils.resource_name("a/b/d/f/f/") == "f" + assert fileutils.resource_name("/a/b/d/f/f.a") == "f.a" + assert fileutils.resource_name("/a/b/d/f/f.a/") == "f.a" + assert fileutils.resource_name("a/b/d/f/f.a") == "f.a" + assert fileutils.resource_name("f.a") == "f.a" + + @skipIf(on_windows, "Windows FS encoding is ... different!") def test_fsdecode_and_fsencode_are_idempotent(self): - a = b'foo\xb1bar' - b = u'foo\udcb1bar' + a = b"foo\xb1bar" + b = "foo\udcb1bar" assert os.fsencode(os.fsdecode(a)) == a assert os.fsencode(os.fsdecode(b)) == a assert os.fsdecode(os.fsencode(a)) == b @@ -284,62 +286,66 @@ def test_fsdecode_and_fsencode_are_idempotent(self): class TestFileUtilsWalk(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_os_walk_with_unicode_path(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") test_dir = str(test_dir) result = list(os.walk(test_dir)) expected = [ - (str(test_dir), ['a'], [u'2.csv']), - (str(test_dir) + sep + 'a', [], [u'gru\u0308n.png']) + (str(test_dir), ["a"], ["2.csv"]), + (str(test_dir) + sep + "a", [], ["gru\u0308n.png"]), ] assert result == expected def test_fileutils_walk(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = [(as_posixpath(t.replace(base, '')), d, sorted(f),) - for t, d, f in fileutils.walk(test_dir)] + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = [ + ( + as_posixpath(t.replace(base, "")), + d, + sorted(f), + ) + for t, d, f in fileutils.walk(test_dir) + ] expected = [ - ('/walk', ['d1'], ['f', 'unicode.zip']), - ('/walk/d1', ['d2'], ['f1']), - ('/walk/d1/d2', ['d3'], ['f2']), - ('/walk/d1/d2/d3', [], ['f3']) + ("/walk", ["d1"], ["f", "unicode.zip"]), + ("/walk/d1", ["d2"], ["f1"]), + ("/walk/d1/d2", ["d3"], ["f2"]), + ("/walk/d1/d2/d3", [], ["f3"]), ] assert result == expected def test_fileutils_walk_with_unicode_path(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") result = list(x[-1] for x in fileutils.walk(test_dir)) - expected = [[u'2.csv'], [u'gru\u0308n.png']] + expected = [["2.csv"], ["gru\u0308n.png"]] assert result == expected def test_fileutils_walk_can_walk_a_single_file(self): - test_file = self.get_test_loc('fileutils/walk/f') + test_file = self.get_test_loc("fileutils/walk/f") result = list(fileutils.walk(test_file)) - expected = [ - (fileutils.parent_directory(test_file), [], ['f']) - ] + expected = [(fileutils.parent_directory(test_file), [], ["f"])] assert result == expected def test_fileutils_walk_can_walk_an_empty_dir(self): test_dir = self.get_temp_dir() result = list(fileutils.walk(test_dir)) - expected = [ - (test_dir, [], []) - ] + expected = [(test_dir, [], [])] assert result == expected - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") if not on_linux: test_dir = str(test_dir) @@ -347,60 +353,58 @@ def test_walk_can_walk_non_utf8_path_from_unicode_path(self): _dirpath, _dirnames, filenames = result assert len(filenames) == 18 - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_os_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(os.walk(test_dir))[0] _dirpath, _dirnames, filenames = result assert len(filenames) == 18 - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_walk_on_symlinks(self): - test_dir = self.get_test_loc('symlink/walk', copy=True) + test_dir = self.get_test_loc("symlink/walk", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) results = list(fileutils.walk(test_link, follow_symlinks=True)) - results = [(os.path.basename(top), dirs, files) - for top, dirs, files in results] - expected = [ - ('test-dir-link', ['dir'], ['a']), - ('dir', [], ['b']) - ] + results = [(os.path.basename(top), dirs, files) for top, dirs, files in results] + expected = [("test-dir-link", ["dir"], ["a"]), ("dir", [], ["b"])] assert results == expected class TestFileUtilsIter(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_resource_iter(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = [as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=False)] + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=False) + ] expected = [ - '/walk/f', - '/walk/unicode.zip', - '/walk/d1/f1', - '/walk/d1/d2/f2', - '/walk/d1/d2/d3/f3' + "/walk/f", + "/walk/unicode.zip", + "/walk/d1/f1", + "/walk/d1/d2/f2", + "/walk/d1/d2/d3/f3", ] assert sorted(result) == sorted(expected) def test_resource_iter_can_iterate_a_single_file(self): - test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter( - test_file, with_dirs=False)] + test_file = self.get_test_loc("fileutils/walk/f") + result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=False)] expected = [as_posixpath(test_file)] assert result == expected def test_resource_iter_can_iterate_a_single_file_with_dirs(self): - test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter( - test_file, with_dirs=True)] + test_file = self.get_test_loc("fileutils/walk/f") + result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=True)] expected = [as_posixpath(test_file)] assert result == expected @@ -417,231 +421,235 @@ def test_resource_iter_can_walk_an_empty_dir_with_dirs(self): assert result == expected def test_resource_iter_without_dir(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=False)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=False) + ] + ) expected = [ - '/walk/f', - '/walk/unicode.zip', - '/walk/d1/f1', - '/walk/d1/d2/f2', - '/walk/d1/d2/d3/f3' + "/walk/f", + "/walk/unicode.zip", + "/walk/d1/f1", + "/walk/d1/d2/f2", + "/walk/d1/d2/d3/f3", ] assert sorted(result) == sorted(expected) def test_resource_iter_with_dirs(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - '/walk/d1', - '/walk/d1/d2', - '/walk/d1/d2/d3', - '/walk/d1/d2/d3/f3', - '/walk/d1/d2/f2', - '/walk/d1/f1', - '/walk/f', - '/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) def test_resource_iter_return_byte_on_byte_input(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - '/walk/d1', - '/walk/d1/d2', - '/walk/d1/d2/d3', - '/walk/d1/d2/d3/f3', - '/walk/d1/d2/f2', - '/walk/d1/f1', - '/walk/f', - '/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) assert all(isinstance(p, str) for p in result) def test_resource_iter_return_unicode_on_unicode_input(self): - test_dir = self.get_test_loc('fileutils/walk') - base = str(self.get_test_loc('fileutils')) - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = str(self.get_test_loc("fileutils")) + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - u'/walk/d1', - u'/walk/d1/d2', - u'/walk/d1/d2/d3', - u'/walk/d1/d2/d3/f3', - u'/walk/d1/d2/f2', - u'/walk/d1/f1', - u'/walk/f', - u'/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) assert all(isinstance(p, str) for p in result) def test_resource_iter_can_walk_unicode_path_with_zip(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") test_dir = str(test_dir) - result = sorted([p.replace(test_dir, '') - for p in fileutils.resource_iter(test_dir)]) + result = sorted([p.replace(test_dir, "") for p in fileutils.resource_iter(test_dir)]) if on_linux: - expected = [ - u'/2.csv', - u'/a', - u'/a/gru\u0308n.png' - ] + expected = ["/2.csv", "/a", "/a/gru\u0308n.png"] elif on_mac: - expected = [ - u'/2.csv', - u'/a', - u'/a/gru\u0308n.png' - ] + expected = ["/2.csv", "/a", "/a/gru\u0308n.png"] elif on_windows: - expected = [ - u'\\2.csv', - u'\\a', - u'\\a\\gru\u0308n.png' - ] + expected = ["\\2.csv", "\\a", "\\a\\gru\u0308n.png"] assert result == expected - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_resource_iter_can_walk_non_utf8_path_from_unicode_path_with_dirs(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(fileutils.resource_iter(test_dir, with_dirs=True)) assert len(result) == 18 - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_resource_iter_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(fileutils.resource_iter(test_dir, with_dirs=False)) assert len(result) == 18 - @skipIf(on_windows, 'Symlinks do not work well on Windows') + @skipIf(on_windows, "Symlinks do not work well on Windows") def test_resource_iter_follow_symlinks(self): - test_dir = self.get_test_loc('symlink/walk', copy=True) + test_dir = self.get_test_loc("symlink/walk", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) - result = [os.path.basename(f) for f in fileutils.resource_iter( - test_dir, follow_symlinks=True)] - expected = [ - 'dir', - 'a', - 'b' + result = [ + os.path.basename(f) for f in fileutils.resource_iter(test_dir, follow_symlinks=True) ] + expected = ["dir", "a", "b"] assert sorted(result) == sorted(expected) class TestBaseName(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_base_name_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'file' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "file" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_dot_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = ".a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_dot_file_with_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_file_with_unknown_composed_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a.tag' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a.tag" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_file_with_known_composed_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tar.gz' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tar.gz" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_dir_path(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "b" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_plain_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'f' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "f" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_plain_file_with_parent_dir_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_dir(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_dir_with_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "f.a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = 'tst' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "tst" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) @@ -649,93 +657,93 @@ def test_file_base_name_on_path_for_plain_file(self): class TestFileName(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_name_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'file' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "file" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = ".a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.a.b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".a.b" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a.tag.gz' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a.tag.gz" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "b" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "f.a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'a.c' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "a.c" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "f.a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = 'tst' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "tst" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) @@ -743,268 +751,259 @@ def test_file_name_on_path_and_location_10(self): class TestFileExtension(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_extension_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".b" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = '.gz' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = ".gz" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = ".a" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = '.c' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = ".c" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_splitext_base(self): - expected = 'path', '.ext' - assert fileutils.splitext('C:\\dir\\path.ext') == expected + expected = "path", ".ext" + assert fileutils.splitext("C:\\dir\\path.ext") == expected def test_splitext_directories_even_with_dotted_names_have_no_extension(self): import ntpath - expected = 'path.ext', '' - assert fileutils.splitext('C:\\dir\\path.ext' + ntpath.sep) == expected - expected = 'path.ext', '' - assert fileutils.splitext('/dir/path.ext/') == expected + expected = "path.ext", "" + assert fileutils.splitext("C:\\dir\\path.ext" + ntpath.sep) == expected + + expected = "path.ext", "" + assert fileutils.splitext("/dir/path.ext/") == expected - expected = 'file', '.txt' - assert fileutils.splitext('/some/file.txt') == expected + expected = "file", ".txt" + assert fileutils.splitext("/some/file.txt") == expected def test_splitext_composite_extensions_for_tarballs_are_properly_handled(self): - expected = 'archive', '.tar.gz' - assert fileutils.splitext('archive.tar.gz') == expected + expected = "archive", ".tar.gz" + assert fileutils.splitext("archive.tar.gz") == expected def test_splitext_name_base(self): - expected = 'path', '.ext' - assert fileutils.splitext_name('path.ext') == expected + expected = "path", ".ext" + assert fileutils.splitext_name("path.ext") == expected def test_splitext_name_directories_have_no_extension(self): - expected = 'path.ext', '' - assert fileutils.splitext_name('path.ext', is_file=False) == expected + expected = "path.ext", "" + assert fileutils.splitext_name("path.ext", is_file=False) == expected - expected = 'file', '.txt' - assert fileutils.splitext_name('file.txt') == expected + expected = "file", ".txt" + assert fileutils.splitext_name("file.txt") == expected def test_splitext_name_composite_extensions_for_tarballs_are_properly_handled(self): - expected = 'archive', '.tar.gz' - assert fileutils.splitext_name('archive.tar.gz') == expected + expected = "archive", ".tar.gz" + assert fileutils.splitext_name("archive.tar.gz") == expected def test_splitext_name_dotfile_are_properly_handled(self): - expected = '.dotfile', '' - assert fileutils.splitext_name('.dotfile') == expected - expected = '.dotfile', '.this' - assert fileutils.splitext_name('.dotfile.this') == expected + expected = ".dotfile", "" + assert fileutils.splitext_name(".dotfile") == expected + expected = ".dotfile", ".this" + assert fileutils.splitext_name(".dotfile.this") == expected class TestParentDir(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_parent_directory_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'a/.a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "a/.a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = 'a/b/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = "a/b/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a/b/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a/b/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'f.a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "f.a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) diff --git a/tests/test_functional.py b/tests/test_functional.py index 0f4b76c..5953c03 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -15,14 +15,12 @@ class TestFunctional(TestCase): - def test_flatten(self): - expected = [7, 6, 5, 4, 'a', 3, 3, 2, 1] + expected = [7, 6, 5, 4, "a", 3, 3, 2, 1] test = flatten([7, (6, [5, [4, ["a"], 3]], 3), 2, 1]) assert test == expected def test_flatten_generator(self): - def gen(): for _ in range(2): yield range(5) @@ -32,17 +30,16 @@ def gen(): assert test == expected def test_flatten_empties(self): - expected = ['a'] - test = flatten([[], (), ['a']]) + expected = ["a"] + test = flatten([[], (), ["a"]]) assert test == expected def test_partial(self): - def test_func(a, b): pass wrapped = partial(test_func, a=2) - assert wrapped.__name__ == 'test_func' + assert wrapped.__name__ == "test_func" def test_memoized(self): call_count = Counter() diff --git a/tests/test_hash.py b/tests/test_hash.py index 0ab15b5..27d2865 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -8,137 +8,157 @@ import os -from commoncode.testcase import FileBasedTesting - from commoncode.hash import b64sha1 from commoncode.hash import checksum from commoncode.hash import get_hasher from commoncode.hash import md5 from commoncode.hash import multi_checksums from commoncode.hash import sha1 +from commoncode.hash import sha1_git from commoncode.hash import sha256 from commoncode.hash import sha512 -from commoncode.hash import sha1_git +from commoncode.testcase import FileBasedTesting class TestHash(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_get_hasher(self): h = get_hasher(160) - assert h(b'a').b64digest() == u'hvfkN_qlp_zhXR3cuerq6jd2Z7g=' - assert h(b'aa').b64digest() == u'4MkDWJjdUvxlxBRUzsnE0mEb-zc=' - assert h(b'aaa').b64digest() == u'fiQN50-x7Qj6CNOAY_amqRRiqBU=' + assert h(b"a").b64digest() == "hvfkN_qlp_zhXR3cuerq6jd2Z7g=" + assert h(b"aa").b64digest() == "4MkDWJjdUvxlxBRUzsnE0mEb-zc=" + assert h(b"aaa").b64digest() == "fiQN50-x7Qj6CNOAY_amqRRiqBU=" def test_short_hashes(self): h = get_hasher(32) - assert h(b'a').hexdigest() == u'0cc175b9' - assert h(b'aa').hexdigest() == u'4124bc0a' + assert h(b"a").hexdigest() == "0cc175b9" + assert h(b"aa").hexdigest() == "4124bc0a" h = get_hasher(64) - assert h(b'aa').hexdigest() == u'4124bc0a9335c27f' + assert h(b"aa").hexdigest() == "4124bc0a9335c27f" def test_sha1_checksum_on_text(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert u'3ca69e8d6c234a469d16ac28a4a658c92267c423' == sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == sha1(test_file) def test_sha1_checksum_on_text2(self): - test_file = self.get_test_loc('hash/dir2/a.txt') - assert u'3ca69e8d6c234a469d16ac28a4a658c92267c423' == sha1(test_file) + test_file = self.get_test_loc("hash/dir2/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == sha1(test_file) def test_sha1_checksum_on_dos_text(self): - test_file = self.get_test_loc('hash/dir2/dos.txt') - assert u'a71718fb198630ae8ba32926015d8555a03cb06c' == sha1(test_file) + test_file = self.get_test_loc("hash/dir2/dos.txt") + assert "a71718fb198630ae8ba32926015d8555a03cb06c" == sha1(test_file) def test_sha1_checksum_base64(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'NKxUZdSKmwT8J18JvCIwZg349Pc=' == b64sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "NKxUZdSKmwT8J18JvCIwZg349Pc=" == b64sha1(test_file) def test_md5_checksum_on_text(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert u'40c53c58fdafacc83cfff6ee3d2f6d69' == md5(test_file) + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == md5(test_file) def test_md5_checksum_on_text2(self): - test_file = self.get_test_loc('hash/dir2/a.txt') - assert u'40c53c58fdafacc83cfff6ee3d2f6d69' == md5(test_file) + test_file = self.get_test_loc("hash/dir2/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == md5(test_file) def test_md5_checksum_on_dos_text(self): - test_file = self.get_test_loc('hash/dir2/dos.txt') - assert u'095f5068940e41df9add5d4cc396c181' == md5(test_file) + test_file = self.get_test_loc("hash/dir2/dos.txt") + assert "095f5068940e41df9add5d4cc396c181" == md5(test_file) def test_md5_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'4760fb467f1ebf3b0aeace4a3926f1a4' == md5(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "4760fb467f1ebf3b0aeace4a3926f1a4" == md5(test_file) def test_sha1_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'34ac5465d48a9b04fc275f09bc2230660df8f4f7' == sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "34ac5465d48a9b04fc275f09bc2230660df8f4f7" == sha1(test_file) def test_sha256_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35' == sha256( - test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35" == sha256( + test_file + ) def test_sha512_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4' == sha512( - test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert ( + "5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4" + == sha512(test_file) + ) def test_checksum_sha1(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert '3ca69e8d6c234a469d16ac28a4a658c92267c423' == checksum( - test_file, 'sha1') + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == checksum(test_file, "sha1") def test_checksum_md5(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert '40c53c58fdafacc83cfff6ee3d2f6d69' == checksum(test_file, 'md5') + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == checksum(test_file, "md5") def test_multi_checksums(self): - test_file = self.get_test_loc('hash/dir1/a.png') - expected = dict([ - ('md5', u'4760fb467f1ebf3b0aeace4a3926f1a4'), - ('sha1', u'34ac5465d48a9b04fc275f09bc2230660df8f4f7'), - ('sha256', u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35'), - ]) - result = multi_checksums(test_file, 'md5 sha1 sha256'.split()) + test_file = self.get_test_loc("hash/dir1/a.png") + expected = dict( + [ + ("md5", "4760fb467f1ebf3b0aeace4a3926f1a4"), + ("sha1", "34ac5465d48a9b04fc275f09bc2230660df8f4f7"), + ("sha256", "1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35"), + ] + ) + result = multi_checksums(test_file, "md5 sha1 sha256".split()) assert result == expected def test_multi_checksums_custom(self): - test_file = self.get_test_loc('hash/dir1/a.png') - result = multi_checksums(test_file, ('sha512',)) - expected = dict([ - ('sha512', u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4'), - ]) + test_file = self.get_test_loc("hash/dir1/a.png") + result = multi_checksums(test_file, ("sha512",)) + expected = dict( + [ + ( + "sha512", + "5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4", + ), + ] + ) assert result == expected def test_multi_checksums_shattered1(self): - test_file = self.get_test_loc('hash/sha1-collision/shattered-1.pdf') - expected = dict([ - ('md5', 'ee4aa52b139d925f8d8884402b0a750c'), - ('sha1', '38762cf7f55934b34d179ae6a4c80cadccbb7f0a'), - ('sha256', '2bb787a73e37352f92383abe7e2902936d1059ad9f1ba6daaa9c1e58ee6970d0'), - ('sha512', '3c19b2cbcf72f7f5b252ea31677b8f2323d6119e49bcc0fb55931d00132385f1e749bb24cbd68c04ac826ae8421802825d3587fe185abf709669bb9693f6b416'), - ('sha1_git', 'ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0'), - ]) + test_file = self.get_test_loc("hash/sha1-collision/shattered-1.pdf") + expected = dict( + [ + ("md5", "ee4aa52b139d925f8d8884402b0a750c"), + ("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a"), + ("sha256", "2bb787a73e37352f92383abe7e2902936d1059ad9f1ba6daaa9c1e58ee6970d0"), + ( + "sha512", + "3c19b2cbcf72f7f5b252ea31677b8f2323d6119e49bcc0fb55931d00132385f1e749bb24cbd68c04ac826ae8421802825d3587fe185abf709669bb9693f6b416", + ), + ("sha1_git", "ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0"), + ] + ) result = multi_checksums(test_file) assert result == expected def test_multi_checksums_shattered2(self): - test_file = self.get_test_loc('hash/sha1-collision/shattered-2.pdf') - expected = dict([ - ('md5', '5bd9d8cabc46041579a311230539b8d1'), - ('sha1', '38762cf7f55934b34d179ae6a4c80cadccbb7f0a'), - ('sha256', 'd4488775d29bdef7993367d541064dbdda50d383f89f0aa13a6ff2e0894ba5ff'), - ('sha512', 'f39a04842e4b28e04558496beb7cb84654ded9c00b2f873c3ef64f9dfdbc760cd0273b816858ba5b203c0dd71af8b65d6a0c1032e00e48ace0b4705eedcc1bab'), - # Note: this is not the same as the sha1_git for shattered-1.pdf ;) - ('sha1_git', 'b621eeccd5c7edac9b7dcba35a8d5afd075e24f2'), - ]) + test_file = self.get_test_loc("hash/sha1-collision/shattered-2.pdf") + expected = dict( + [ + ("md5", "5bd9d8cabc46041579a311230539b8d1"), + ("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a"), + ("sha256", "d4488775d29bdef7993367d541064dbdda50d383f89f0aa13a6ff2e0894ba5ff"), + ( + "sha512", + "f39a04842e4b28e04558496beb7cb84654ded9c00b2f873c3ef64f9dfdbc760cd0273b816858ba5b203c0dd71af8b65d6a0c1032e00e48ace0b4705eedcc1bab", + ), + # Note: this is not the same as the sha1_git for shattered-1.pdf ;) + ("sha1_git", "b621eeccd5c7edac9b7dcba35a8d5afd075e24f2"), + ] + ) result = multi_checksums(test_file) assert result == expected def test_sha1_git_checksum(self): # $ pushd tests/commoncode/data && for f in `find hash/ -type f` ; # do echo -n "$f ";git hash-object --literally $f; done && popd - tests = [t.strip().split() for t in ''' + tests = [ + t.strip().split() + for t in """ hash/dir1/a.txt de980441c3ab03a8c07dda1ad27b8a11f39deb1e hash/dir1/a.png 5f212358671a3ada8794cb14fb5227f596447a8c hash/sha1-collision/shattered-1.pdf ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0 @@ -147,7 +167,9 @@ def test_sha1_git_checksum(self): hash/sha1-collision/shattered-2.pdf b621eeccd5c7edac9b7dcba35a8d5afd075e24f2 hash/dir2/dos.txt 0d2d3a69833f1ebcbf420875cfbc93f132bc8a0b hash/dir2/a.txt de980441c3ab03a8c07dda1ad27b8a11f39deb1e - '''.splitlines() if t.strip()] + """.splitlines() + if t.strip() + ] for test_file, expected_sha1_git in tests: test_file = self.get_test_loc(test_file) # test that we match the git hash-object diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 984c512..8c74286 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -17,94 +17,98 @@ class IgnoreTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_eclipse1(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.settings') + test = os.path.join(test_base, ".settings") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse2(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.settings/somefile') + test = os.path.join(test_base, ".settings/somefile") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse3(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.project') + test = os.path.join(test_base, ".project") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse4(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.pydevproject') + test = os.path.join(test_base, ".pydevproject") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac1(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '__MACOSX') + test = os.path.join(test_base, "__MACOSX") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac2(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join( - test_base, '__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv') + test = os.path.join(test_base, "__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac3(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '.DS_Store') + test = os.path.join(test_base, ".DS_Store") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac4(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '.DS_Store/a') + test = os.path.join(test_base, ".DS_Store/a") assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_mac5(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '._.DS_Store') + test = os.path.join(test_base, "._.DS_Store") # this is really weird as a behavior # 'Default ignore: MacOSX artifact' assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_msft(self): - test_dir = self.extract_test_tar('ignore/excludes/msft-vs.tgz') - test = os.path.join(test_dir, 'msft-vs/tst.sluo') + test_dir = self.extract_test_tar("ignore/excludes/msft-vs.tgz") + test = os.path.join(test_dir, "msft-vs/tst.sluo") # 'Default ignore: Microsoft VS project artifact' ?? assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_skip_vcs_files_and_dirs(self): - test_dir = self.extract_test_tar('ignore/vcs.tgz') + test_dir = self.extract_test_tar("ignore/vcs.tgz") result = [] for top, dirs, files in os.walk(test_dir, topdown=True): not_ignored = [] for d in dirs: p = os.path.join(top, d) ign = ignore.is_ignored(p, ignore.default_ignores, {}) - tp = fileutils.as_posixpath(p.replace(test_dir, '')) - result.append((tp, ign,)) + tp = fileutils.as_posixpath(p.replace(test_dir, "")) + result.append( + ( + tp, + ign, + ) + ) if not ign: not_ignored.append(d) @@ -114,94 +118,90 @@ def test_is_ignored_skip_vcs_files_and_dirs(self): for f in files: p = os.path.join(top, f) ign = ignore.is_ignored(p, ignore.default_ignores, {}) - tp = fileutils.as_posixpath(p.replace(test_dir, '')) - result.append((tp, ign,)) + tp = fileutils.as_posixpath(p.replace(test_dir, "")) + result.append( + ( + tp, + ign, + ) + ) expected = [ - ('/vcs', False), - ('/vcs/.bzr', True), - ('/vcs/.git', True), - ('/vcs/.hg', True), - ('/vcs/.repo', True), - ('/vcs/.svn', True), - ('/vcs/CVS', True), - ('/vcs/_darcs', True), - ('/vcs/_MTN', True), - ('/vcs/.bzrignore', True), - ('/vcs/.cvsignore', True), - ('/vcs/.gitignore', True), - ('/vcs/.hgignore', True), - ('/vcs/.svnignore', True), - ('/vcs/vssver.scc', True), + ("/vcs", False), + ("/vcs/.bzr", True), + ("/vcs/.git", True), + ("/vcs/.hg", True), + ("/vcs/.repo", True), + ("/vcs/.svn", True), + ("/vcs/CVS", True), + ("/vcs/_darcs", True), + ("/vcs/_MTN", True), + ("/vcs/.bzrignore", True), + ("/vcs/.cvsignore", True), + ("/vcs/.gitignore", True), + ("/vcs/.hgignore", True), + ("/vcs/.svnignore", True), + ("/vcs/vssver.scc", True), ] assert sorted(result) == sorted(expected) def test_fileset_is_included_with_default_ignore_does_not_skip_one_char_names(self): # use fileset directly to work on strings not locations from commoncode import fileset - tests = [c for c in 'HFS+ Private Data'] + 'HFS+ Private Data'.split() - result = [(t, - fileset.is_included(t, excludes=ignore.default_ignores, includes={})) - for t in tests] + + tests = [c for c in "HFS+ Private Data"] + "HFS+ Private Data".split() + result = [ + (t, fileset.is_included(t, excludes=ignore.default_ignores, includes={})) for t in tests + ] expected = [ - ('H', True), - ('F', True), - ('S', True), - ('+', True), - (' ', False), - ('P', True), - ('r', True), - ('i', True), - ('v', True), - ('a', True), - ('t', True), - ('e', True), - (' ', False), - ('D', True), - ('a', True), - ('t', True), - ('a', True), - ('HFS+', True), - ('Private', True), - ('Data', True) + ("H", True), + ("F", True), + ("S", True), + ("+", True), + (" ", False), + ("P", True), + ("r", True), + ("i", True), + ("v", True), + ("a", True), + ("t", True), + ("e", True), + (" ", False), + ("D", True), + ("a", True), + ("t", True), + ("a", True), + ("HFS+", True), + ("Private", True), + ("Data", True), ] assert result == expected - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_path_string_skip_special(self): - test_path = '/test/path' - assert ignore.is_ignored( - test_path, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_path, {'asdf': 'skip'}, {}, skip_special=False) + test_path = "/test/path" + assert ignore.is_ignored(test_path, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_path, {"asdf": "skip"}, {}, skip_special=False) - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_special_files_skip_special(self): test_fifo = self.get_temp_file() os.mkfifo(test_fifo) - assert ignore.is_ignored( - test_fifo, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_fifo, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored(test_fifo, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_fifo, {"asdf": "skip"}, {}, skip_special=False) test_symlink = self.get_temp_file() - test_file_location = self.get_test_loc('ignore/vcs.tgz') + test_file_location = self.get_test_loc("ignore/vcs.tgz") os.symlink(test_file_location, test_symlink) - assert ignore.is_ignored( - test_symlink, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_symlink, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored(test_symlink, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_symlink, {"asdf": "skip"}, {}, skip_special=False) - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_real_location_skip_special(self): - test_file_location = self.get_test_loc('ignore/vcs.tgz') - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=False) - - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=False) + test_file_location = self.get_test_loc("ignore/vcs.tgz") + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=False) + + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=False) diff --git a/tests/test_paths.py b/tests/test_paths.py index 5d7d94d..67bff07 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -12,316 +12,315 @@ class TestPortablePath(TestCase): - def test_safe_path_mixed_slashes(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js" assert test == expected def test_safe_path_mixed_slashes_and_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js" assert test == expected def test_safe_path_windows_style(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h') - expected = 'C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h' + "C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h" + ) + expected = "C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h" assert test == expected def test_safe_path_windows_style_mixed_slashes_no_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css" assert test == expected def test_safe_path_windows_style_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css" assert test == expected def test_safe_path_windows_style_posix_slashes(self): test = paths.safe_path( - 'C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java') - expected = 'C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' + "C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java" + ) + expected = "C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java" assert test == expected def test_safe_path_windows_style_relative(self): - test = paths.safe_path('includes\\webform.components.inc') - expected = 'includes/webform.components.inc' + test = paths.safe_path("includes\\webform.components.inc") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_windows_style_absolute_trailing_slash(self): - test = paths.safe_path('\\includes\\webform.components.inc\\') - expected = 'includes/webform.components.inc' + test = paths.safe_path("\\includes\\webform.components.inc\\") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_relative(self): - test = paths.safe_path('includes/webform.components.inc') - expected = 'includes/webform.components.inc' + test = paths.safe_path("includes/webform.components.inc") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_absolute_trailing_slash(self): - test = paths.safe_path('/includes/webform.components.inc/') - expected = 'includes/webform.components.inc' + test = paths.safe_path("/includes/webform.components.inc/") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_french_char(self): - test = paths.safe_path('/includes/webform.compon\xc3nts.inc/') - expected = 'includes/webform.componAnts.inc' + test = paths.safe_path("/includes/webform.compon\xc3nts.inc/") + expected = "includes/webform.componAnts.inc" assert test == expected def test_safe_path_posix_style_chinese_char(self): - test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/') - expected = 'includes/webform.componNSnts.inc' - assert test == expected + test = paths.safe_path(b"/includes/webform.compon\xd2\xaants.inc/") + expected = [ + "includes/webform.componNSnts.inc", + "includes/webform.componS_nts.inc", + ] + assert test in expected def test_safe_path_windows_style_dots(self): - test = paths.safe_path('\\includes\\..\\webform.components.inc\\') - expected = 'webform.components.inc' + test = paths.safe_path("\\includes\\..\\webform.components.inc\\") + expected = "webform.components.inc" assert test == expected def test_safe_path_windows_style_many_dots(self): - test = paths.safe_path( - '.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') - expected = 'dotdot/dotdot/webform.components.inc' + test = paths.safe_path(".\\includes\\.\\..\\..\\..\\webform.components.inc\\.") + expected = "dotdot/dotdot/webform.components.inc" assert test == expected def test_safe_path_posix_style_dots(self): - test = paths.safe_path('includes/../webform.components.inc') - expected = 'webform.components.inc' + test = paths.safe_path("includes/../webform.components.inc") + expected = "webform.components.inc" assert test == expected def test_safe_path_posix_style_many_dots(self): - test = paths.safe_path( - './includes/./../../../../webform.components.inc/.') - expected = 'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.safe_path("./includes/./../../../../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_safe_path_posix_only(self): - test_path = 'var/lib/dpkg/info/libgsm1:amd64.list' + test_path = "var/lib/dpkg/info/libgsm1:amd64.list" test = paths.safe_path(test_path) - expected = 'var/lib/dpkg/info/libgsm1_amd64.list' + expected = "var/lib/dpkg/info/libgsm1_amd64.list" assert test == expected test = paths.safe_path(test_path, posix_only=True) assert test == test_path def test_resolve_mixed_slash(self): - test = paths.resolve('C:\\..\\./drupal.js') - expected = 'C/drupal.js' + test = paths.resolve("C:\\..\\./drupal.js") + expected = "C/drupal.js" assert test == expected def test_resolve_2(self): - test = paths.resolve('\\includes\\..\\webform.components.inc\\') - expected = 'webform.components.inc' + test = paths.resolve("\\includes\\..\\webform.components.inc\\") + expected = "webform.components.inc" assert test == expected def test_resolve_3(self): - test = paths.resolve('includes/../webform.components.inc') - expected = 'webform.components.inc' + test = paths.resolve("includes/../webform.components.inc") + expected = "webform.components.inc" assert test == expected def test_resolve_4(self): - test = paths.resolve( - '////.//includes/./../..//..///../webform.components.inc/.') - expected = 'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.resolve("////.//includes/./../..//..///../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_resolve_5(self): - test = paths.resolve( - u'////.//includes/./../..//..///../webform.components.inc/.') - expected = u'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.resolve("////.//includes/./../..//..///../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_resolve_6(self): - test = paths.resolve('includes/../') - expected = '.' + test = paths.resolve("includes/../") + expected = "." assert test == expected def test_portable_filename(self): - expected = 'A___file__with_Spaces.mov' - assert paths.portable_filename( - "A:\\ file/ with Spaces.mov") == expected + expected = "A___file__with_Spaces.mov" + assert paths.portable_filename("A:\\ file/ with Spaces.mov") == expected # Test `preserve_spaces` option. Spaces should not be replaced - expected = 'Program Files (x86)' - assert paths.portable_filename( - "Program Files (x86)", preserve_spaces=True) == expected + expected = "Program Files (x86)" + assert paths.portable_filename("Program Files (x86)", preserve_spaces=True) == expected # Unresolved relative paths will be treated as a single filename. Use # resolve instead if you want to resolve paths: - expected = '___.._.._etc_passwd' + expected = "___.._.._etc_passwd" assert paths.portable_filename("../../../etc/passwd") == expected # Unicode name are transliterated: - expected = 'This_contain_UMLAUT_umlauts.txt' - assert paths.portable_filename( - u'This contain UMLAUT \xfcml\xe4uts.txt') == expected + expected = "This_contain_UMLAUT_umlauts.txt" + assert paths.portable_filename("This contain UMLAUT \xfcml\xe4uts.txt") == expected # Check to see if illegal Windows filenames are properly handled for illegal_window_name in paths.ILLEGAL_WINDOWS_NAMES: # Rename files with names that are illegal on Windows - expected = f'{illegal_window_name}_' + expected = f"{illegal_window_name}_" assert paths.portable_filename(illegal_window_name) == expected # Allow files with names that are illegal on Windows - assert paths.portable_filename( - illegal_window_name, posix_only=True) == illegal_window_name + assert ( + paths.portable_filename(illegal_window_name, posix_only=True) == illegal_window_name + ) # Check to see if the posix_only option does and does not replace # punctuation characters that are illegal in Windows filenames for valid_posix_path_char in paths.posix_legal_punctuation: - test_name = f'test{valid_posix_path_char}' - assert paths.portable_filename( - test_name, posix_only=True) == test_name + test_name = f"test{valid_posix_path_char}" + assert paths.portable_filename(test_name, posix_only=True) == test_name if valid_posix_path_char not in paths.legal_punctuation: - expected = f'test_' + expected = f"test_" assert paths.portable_filename(test_name) == expected class TestCommonPath(TestCase): - def test_common_path_prefix1(self): - test = paths.common_path_prefix('/a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_prefix("/a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_prefix2(self): - test = paths.common_path_prefix('/a/b/c', '/a/b') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c", "/a/b") + assert test == ("a/b", 2) def test_common_path_prefix3(self): - test = paths.common_path_prefix('/a/b', '/a/b/c') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b", "/a/b/c") + assert test == ("a/b", 2) def test_common_path_prefix4(self): - test = paths.common_path_prefix('/a', '/a') - assert test == ('a', 1) + test = paths.common_path_prefix("/a", "/a") + assert test == ("a", 1) def test_common_path_prefix_path_root(self): - test = paths.common_path_prefix('/a/b/c', '/') + test = paths.common_path_prefix("/a/b/c", "/") assert test == (None, 0) def test_common_path_prefix_root_path(self): - test = paths.common_path_prefix('/', '/a/b/c') + test = paths.common_path_prefix("/", "/a/b/c") assert test == (None, 0) def test_common_path_prefix_root_root(self): - test = paths.common_path_prefix('/', '/') + test = paths.common_path_prefix("/", "/") assert test == (None, 0) def test_common_path_prefix_path_elements_are_similar(self): - test = paths.common_path_prefix('/a/b/c', '/a/b/d') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c", "/a/b/d") + assert test == ("a/b", 2) def test_common_path_prefix_no_match(self): - test = paths.common_path_prefix('/abc/d', '/abe/f') + test = paths.common_path_prefix("/abc/d", "/abe/f") assert test == (None, 0) def test_common_path_prefix_ignore_training_slashes(self): - test = paths.common_path_prefix('/a/b/c/', '/a/b/c/') - assert test == ('a/b/c', 3) + test = paths.common_path_prefix("/a/b/c/", "/a/b/c/") + assert test == ("a/b/c", 3) def test_common_path_prefix8(self): - test = paths.common_path_prefix('/a/b/c/', '/a/b') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c/", "/a/b") + assert test == ("a/b", 2) def test_common_path_prefix10(self): - test = paths.common_path_prefix('/a/b/c.txt', '/a/b/b.txt') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c.txt", "/a/b/b.txt") + assert test == ("a/b", 2) def test_common_path_prefix11(self): - test = paths.common_path_prefix('/a/b/c.txt', '/a/b.txt') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/b/c.txt", "/a/b.txt") + assert test == ("a", 1) def test_common_path_prefix12(self): - test = paths.common_path_prefix('/a/c/e/x.txt', '/a/d/a.txt') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/x.txt", "/a/d/a.txt") + assert test == ("a", 1) def test_common_path_prefix13(self): - test = paths.common_path_prefix('/a/c/e/x.txt', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/x.txt", "/a/d/") + assert test == ("a", 1) def test_common_path_prefix14(self): - test = paths.common_path_prefix('/a/c/e/', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/", "/a/d/") + assert test == ("a", 1) def test_common_path_prefix15(self): - test = paths.common_path_prefix('/a/c/e/', '/a/c/a.txt') - assert test == ('a/c', 2) + test = paths.common_path_prefix("/a/c/e/", "/a/c/a.txt") + assert test == ("a/c", 2) def test_common_path_prefix16(self): - test = paths.common_path_prefix('/a/c/e/', '/a/c/f/') - assert test == ('a/c', 2) + test = paths.common_path_prefix("/a/c/e/", "/a/c/f/") + assert test == ("a/c", 2) def test_common_path_prefix17(self): - test = paths.common_path_prefix('/a/a.txt', '/a/b.txt/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/a.txt", "/a/b.txt/") + assert test == ("a", 1) def test_common_path_prefix18(self): - test = paths.common_path_prefix('/a/c/', '/a/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/", "/a/") + assert test == ("a", 1) def test_common_path_prefix19(self): - test = paths.common_path_prefix('/a/c.txt', '/a/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c.txt", "/a/") + assert test == ("a", 1) def test_common_path_prefix20(self): - test = paths.common_path_prefix('/a/c/', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/", "/a/d/") + assert test == ("a", 1) def test_common_path_suffix(self): - test = paths.common_path_suffix('/a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_suffix("/a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_suffix_absolute_relative(self): - test = paths.common_path_suffix('a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_suffix("a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_suffix_find_subpath(self): - test = paths.common_path_suffix('/z/b/c', '/a/b/c') - assert test == ('b/c', 2) + test = paths.common_path_suffix("/z/b/c", "/a/b/c") + assert test == ("b/c", 2) def test_common_path_suffix_handles_relative_path(self): - test = paths.common_path_suffix('a/b', 'a/b') - assert test == ('a/b', 2) + test = paths.common_path_suffix("a/b", "a/b") + assert test == ("a/b", 2) def test_common_path_suffix_handles_relative_subpath(self): - test = paths.common_path_suffix('zsds/adsds/a/b/b/c', 'a//a/d//b/c') - assert test == ('b/c', 2) + test = paths.common_path_suffix("zsds/adsds/a/b/b/c", "a//a/d//b/c") + assert test == ("b/c", 2) def test_common_path_suffix_ignore_and_strip_trailing_slash(self): - test = paths.common_path_suffix('zsds/adsds/a/b/b/c/', 'a//a/d//b/c/') - assert test == ('b/c', 2) + test = paths.common_path_suffix("zsds/adsds/a/b/b/c/", "a//a/d//b/c/") + assert test == ("b/c", 2) def test_common_path_suffix_return_None_if_no_common_suffix(self): - test = paths.common_path_suffix('/a/b/c', '/') + test = paths.common_path_suffix("/a/b/c", "/") assert test == (None, 0) def test_common_path_suffix_return_None_if_no_common_suffix2(self): - test = paths.common_path_suffix('/', '/a/b/c') + test = paths.common_path_suffix("/", "/a/b/c") assert test == (None, 0) def test_common_path_suffix_match_only_whole_segments(self): # only segments are honored, commonality within segment is ignored - test = paths.common_path_suffix( - 'this/is/aaaa/great/path', 'this/is/aaaaa/great/path') - assert test == ('great/path', 2) + test = paths.common_path_suffix("this/is/aaaa/great/path", "this/is/aaaaa/great/path") + assert test == ("great/path", 2) def test_common_path_suffix_two_root(self): - test = paths.common_path_suffix('/', '/') + test = paths.common_path_suffix("/", "/") assert test == (None, 0) def test_common_path_suffix_empty_root(self): - test = paths.common_path_suffix('', '/') + test = paths.common_path_suffix("", "/") assert test == (None, 0) def test_common_path_suffix_root_empty(self): - test = paths.common_path_suffix('/', '') + test = paths.common_path_suffix("/", "") assert test == (None, 0) def test_common_path_suffix_empty_empty(self): - test = paths.common_path_suffix('', '') + test = paths.common_path_suffix("", "") assert test == (None, 0) diff --git a/tests/test_resource.py b/tests/test_resource.py index 25f2bd9..9711b66 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -24,21 +24,21 @@ class TestCodebase(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_walk_defaults(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk()) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected @@ -47,79 +47,79 @@ def test_Codebase_do_not_ignore_by_default_older_sccs_and_rcs_dirs(self): from commoncode.fileutils import create_dir test_codebase = self.get_temp_dir() - create_dir(join(test_codebase, 'sccs', 'a')) - create_dir(join(test_codebase, 'rcs', 'b')) + create_dir(join(test_codebase, "sccs", "a")) + create_dir(join(test_codebase, "rcs", "b")) codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=True, skip_root=True)) - expected = ['rcs', 'b', 'sccs', 'a'] + expected = ["rcs", "b", "sccs", "a"] assert [r.name for r in results] == expected def test_walk_topdown(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=True)) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_bottomup(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=False)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('dir', False), - ('file', True), - ('other dir', False), - ('codebase', False), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("dir", False), + ("file", True), + ("other dir", False), + ("codebase", False), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_basic(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_filtered_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) results = list(codebase.walk_filtered()) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_all_filtered(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -130,14 +130,14 @@ def test_walk_filtered_with_all_filtered(self): assert [(r.name, r.is_file) for r in results] == expected def test_compute_counts_filtered_None(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = codebase.compute_counts(skip_filtered=True) expected = (5, 3, 0) assert results == expected def test_compute_counts_filtered_None_with_size(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if res.is_file: @@ -149,14 +149,14 @@ def test_compute_counts_filtered_None_with_size(self): assert results == expected def test_compute_counts_filtered_None_with_cache(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = codebase.compute_counts(skip_filtered=True) expected = (5, 3, 0) assert results == expected def test_compute_counts_filtered_all(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -166,7 +166,7 @@ def test_compute_counts_filtered_all(self): assert results == expected def test_compute_counts_filtered_all_with_cache(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -176,7 +176,7 @@ def test_compute_counts_filtered_all_with_cache(self): assert results == expected def test_compute_counts_filtered_files(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if res.is_file: @@ -187,7 +187,7 @@ def test_compute_counts_filtered_files(self): assert results == expected def test_compute_counts_filtered_dirs(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if not res.is_file: @@ -198,7 +198,7 @@ def test_compute_counts_filtered_dirs(self): assert results == expected def test_walk_filtered_dirs(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if not res.is_file: @@ -207,33 +207,33 @@ def test_walk_filtered_dirs(self): results = list(codebase.walk_filtered(topdown=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_skip_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) results = list(codebase.walk_filtered(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_all_skip_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -243,21 +243,21 @@ def test_walk_filtered_all_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) results = list(codebase.walk(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_skip_root_and_single_file_not_filtered(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered__with_skip_root_and_filtered_single_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) @@ -266,55 +266,51 @@ def test_walk_filtered__with_skip_root_and_filtered_single_file(self): assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_file_with_children(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase, strip_root=True) root = codebase.root - c1 = codebase._get_or_create_resource( - 'child1', parent=root, is_file=True) - codebase._get_or_create_resource('child2', parent=c1, is_file=False) + c1 = codebase._get_or_create_resource("child1", parent=root, is_file=True) + codebase._get_or_create_resource("child2", parent=c1, is_file=False) results = list(codebase.walk(skip_root=True)) - expected = [('et131x.h', True), ('child1', True), ('child2', False)] + expected = [("et131x.h", True), ("child1", True), ("child2", False)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_skip_root_and_single_file_with_children(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase, strip_root=True) - c1 = codebase._get_or_create_resource( - 'some child', parent=codebase.root, is_file=True) - c2 = codebase._get_or_create_resource( - 'some child2', parent=c1, is_file=False) + c1 = codebase._get_or_create_resource("some child", parent=codebase.root, is_file=True) + c2 = codebase._get_or_create_resource("some child2", parent=c1, is_file=False) c2.is_filtered = True codebase.save_resource(c2) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected c1.is_filtered = True codebase.save_resource(c1) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_dir(self): - test_codebase = self.get_temp_dir('walk') + test_codebase = self.get_temp_dir("walk") codebase = Codebase(test_codebase, strip_root=True) results = list(codebase.walk(skip_root=True)) - expected = [('walk', False)] + expected = [("walk", False)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skipped_directories_should_not_be_yielded(self): # Resources that we continue past should not be added to the result list - test_codebase = self.get_test_loc( - 'resource/skip_directories_during_walk') + test_codebase = self.get_test_loc("resource/skip_directories_during_walk") cdbs = Codebase(test_codebase) def _ignored(resource, codebase): - return resource.is_dir and resource.name == 'skip-this-directory' + return resource.is_dir and resource.name == "skip-this-directory" result = [ res.name @@ -324,47 +320,44 @@ def _ignored(resource, codebase): ) ] - expected = ['skip_directories_during_walk', 'this-should-be-returned'] + expected = ["skip_directories_during_walk", "this-should-be-returned"] assert result == expected def test__create_resource_can_add_child_to_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) - codebase._get_or_create_resource( - 'some child', codebase.root, is_file=True) + codebase._get_or_create_resource("some child", codebase.root, is_file=True) results = list(codebase.walk()) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected def test__create_resource_can_add_child_to_dir(self): - test_codebase = self.get_temp_dir('resource') + test_codebase = self.get_temp_dir("resource") codebase = Codebase(test_codebase) - codebase._get_or_create_resource( - 'some child', codebase.root, is_file=False) + codebase._get_or_create_resource("some child", codebase.root, is_file=False) results = list(codebase.walk()) - expected = [('resource', False), ('some child', False)] + expected = [("resource", False), ("some child", False)] assert [(r.name, r.is_file) for r in results] == expected def test_get_resource_for_single_resource_codebase(self): - test_codebase = self.get_temp_dir('resource') + test_codebase = self.get_temp_dir("resource") codebase = Codebase(test_codebase) - assert not (codebase.root is codebase.get_resource('resource')) - assert codebase.get_resource('resource') == codebase.root + assert not (codebase.root is codebase.get_resource("resource")) + assert codebase.get_resource("resource") == codebase.root def test_get_resource_for_multiple_resource_codebase(self): - test_codebase = self.get_temp_dir('resource') - for name in ('a', 'b', 'c'): - with open(os.path.join(test_codebase, name), 'w') as o: - o.write('\n') + test_codebase = self.get_temp_dir("resource") + for name in ("a", "b", "c"): + with open(os.path.join(test_codebase, name), "w") as o: + o.write("\n") codebase = Codebase(test_codebase) - assert codebase.get_resource('resource/a').path == 'resource/a' - assert codebase.get_resource('/resource/c').path == 'resource/c' - assert codebase.get_resource( - 'resource/dsasda/../b/').path == 'resource/b' + assert codebase.get_resource("resource/a").path == "resource/a" + assert codebase.get_resource("/resource/c").path == "resource/c" + assert codebase.get_resource("resource/dsasda/../b/").path == "resource/b" def test_Resource_build_path(self): - test_dir = self.get_test_loc('resource/samples') + test_dir = self.get_test_loc("resource/samples") locations = [] for top, dirs, files in os.walk(test_dir): for x in dirs: @@ -376,49 +369,49 @@ def test_Resource_build_path(self): resources_no_root = list(codebase.walk(skip_root=True)) expected_default = [ - 'samples/JGroups', - 'samples/zlib', - 'samples/arch', - 'samples/README', - 'samples/screenshot.png', - 'samples/JGroups/src', - 'samples/JGroups/licenses', - 'samples/JGroups/LICENSE', - 'samples/JGroups/EULA', - 'samples/JGroups/src/GuardedBy.java', - 'samples/JGroups/src/ImmutableReference.java', - 'samples/JGroups/src/RouterStub.java', - 'samples/JGroups/src/S3_PING.java', - 'samples/JGroups/src/FixedMembershipToken.java', - 'samples/JGroups/src/RouterStubManager.java', - 'samples/JGroups/src/RATE_LIMITER.java', - 'samples/JGroups/licenses/cpl-1.0.txt', - 'samples/JGroups/licenses/bouncycastle.txt', - 'samples/JGroups/licenses/lgpl.txt', - 'samples/JGroups/licenses/apache-2.0.txt', - 'samples/JGroups/licenses/apache-1.1.txt', - 'samples/zlib/dotzlib', - 'samples/zlib/iostream2', - 'samples/zlib/infback9', - 'samples/zlib/gcc_gvmat64', - 'samples/zlib/ada', - 'samples/zlib/deflate.h', - 'samples/zlib/zutil.c', - 'samples/zlib/zlib.h', - 'samples/zlib/deflate.c', - 'samples/zlib/zutil.h', - 'samples/zlib/adler32.c', - 'samples/zlib/dotzlib/AssemblyInfo.cs', - 'samples/zlib/dotzlib/LICENSE_1_0.txt', - 'samples/zlib/dotzlib/readme.txt', - 'samples/zlib/dotzlib/ChecksumImpl.cs', - 'samples/zlib/iostream2/zstream_test.cpp', - 'samples/zlib/iostream2/zstream.h', - 'samples/zlib/infback9/infback9.c', - 'samples/zlib/infback9/infback9.h', - 'samples/zlib/gcc_gvmat64/gvmat64.S', - 'samples/zlib/ada/zlib.ads', - 'samples/arch/zlib.tar.gz', + "samples/JGroups", + "samples/zlib", + "samples/arch", + "samples/README", + "samples/screenshot.png", + "samples/JGroups/src", + "samples/JGroups/licenses", + "samples/JGroups/LICENSE", + "samples/JGroups/EULA", + "samples/JGroups/src/GuardedBy.java", + "samples/JGroups/src/ImmutableReference.java", + "samples/JGroups/src/RouterStub.java", + "samples/JGroups/src/S3_PING.java", + "samples/JGroups/src/FixedMembershipToken.java", + "samples/JGroups/src/RouterStubManager.java", + "samples/JGroups/src/RATE_LIMITER.java", + "samples/JGroups/licenses/cpl-1.0.txt", + "samples/JGroups/licenses/bouncycastle.txt", + "samples/JGroups/licenses/lgpl.txt", + "samples/JGroups/licenses/apache-2.0.txt", + "samples/JGroups/licenses/apache-1.1.txt", + "samples/zlib/dotzlib", + "samples/zlib/iostream2", + "samples/zlib/infback9", + "samples/zlib/gcc_gvmat64", + "samples/zlib/ada", + "samples/zlib/deflate.h", + "samples/zlib/zutil.c", + "samples/zlib/zlib.h", + "samples/zlib/deflate.c", + "samples/zlib/zutil.h", + "samples/zlib/adler32.c", + "samples/zlib/dotzlib/AssemblyInfo.cs", + "samples/zlib/dotzlib/LICENSE_1_0.txt", + "samples/zlib/dotzlib/readme.txt", + "samples/zlib/dotzlib/ChecksumImpl.cs", + "samples/zlib/iostream2/zstream_test.cpp", + "samples/zlib/iostream2/zstream.h", + "samples/zlib/infback9/infback9.c", + "samples/zlib/infback9/infback9.h", + "samples/zlib/gcc_gvmat64/gvmat64.S", + "samples/zlib/ada/zlib.ads", + "samples/arch/zlib.tar.gz", ] default = sorted( @@ -427,49 +420,49 @@ def test_Resource_build_path(self): assert default == sorted(expected_default) expected_strip_root = [ - 'JGroups', - 'zlib', - 'arch', - 'README', - 'screenshot.png', - 'JGroups/src', - 'JGroups/licenses', - 'JGroups/LICENSE', - 'JGroups/EULA', - 'JGroups/src/GuardedBy.java', - 'JGroups/src/ImmutableReference.java', - 'JGroups/src/RouterStub.java', - 'JGroups/src/S3_PING.java', - 'JGroups/src/FixedMembershipToken.java', - 'JGroups/src/RouterStubManager.java', - 'JGroups/src/RATE_LIMITER.java', - 'JGroups/licenses/cpl-1.0.txt', - 'JGroups/licenses/bouncycastle.txt', - 'JGroups/licenses/lgpl.txt', - 'JGroups/licenses/apache-2.0.txt', - 'JGroups/licenses/apache-1.1.txt', - 'zlib/dotzlib', - 'zlib/iostream2', - 'zlib/infback9', - 'zlib/gcc_gvmat64', - 'zlib/ada', - 'zlib/deflate.h', - 'zlib/zutil.c', - 'zlib/zlib.h', - 'zlib/deflate.c', - 'zlib/zutil.h', - 'zlib/adler32.c', - 'zlib/dotzlib/AssemblyInfo.cs', - 'zlib/dotzlib/LICENSE_1_0.txt', - 'zlib/dotzlib/readme.txt', - 'zlib/dotzlib/ChecksumImpl.cs', - 'zlib/iostream2/zstream_test.cpp', - 'zlib/iostream2/zstream.h', - 'zlib/infback9/infback9.c', - 'zlib/infback9/infback9.h', - 'zlib/gcc_gvmat64/gvmat64.S', - 'zlib/ada/zlib.ads', - 'arch/zlib.tar.gz', + "JGroups", + "zlib", + "arch", + "README", + "screenshot.png", + "JGroups/src", + "JGroups/licenses", + "JGroups/LICENSE", + "JGroups/EULA", + "JGroups/src/GuardedBy.java", + "JGroups/src/ImmutableReference.java", + "JGroups/src/RouterStub.java", + "JGroups/src/S3_PING.java", + "JGroups/src/FixedMembershipToken.java", + "JGroups/src/RouterStubManager.java", + "JGroups/src/RATE_LIMITER.java", + "JGroups/licenses/cpl-1.0.txt", + "JGroups/licenses/bouncycastle.txt", + "JGroups/licenses/lgpl.txt", + "JGroups/licenses/apache-2.0.txt", + "JGroups/licenses/apache-1.1.txt", + "zlib/dotzlib", + "zlib/iostream2", + "zlib/infback9", + "zlib/gcc_gvmat64", + "zlib/ada", + "zlib/deflate.h", + "zlib/zutil.c", + "zlib/zlib.h", + "zlib/deflate.c", + "zlib/zutil.h", + "zlib/adler32.c", + "zlib/dotzlib/AssemblyInfo.cs", + "zlib/dotzlib/LICENSE_1_0.txt", + "zlib/dotzlib/readme.txt", + "zlib/dotzlib/ChecksumImpl.cs", + "zlib/iostream2/zstream_test.cpp", + "zlib/iostream2/zstream.h", + "zlib/infback9/infback9.c", + "zlib/infback9/infback9.h", + "zlib/gcc_gvmat64/gvmat64.S", + "zlib/ada/zlib.ads", + "arch/zlib.tar.gz", ] stripped = sorted(r.strip_root_path for r in resources_no_root) assert stripped == sorted(expected_strip_root) @@ -480,7 +473,7 @@ def test_Resource_build_path(self): assert full_loc.endswith((ending)) def test_compute_counts_when_using_disk_cache(self): - test_codebase = self.get_test_loc('resource/samples') + test_codebase = self.get_test_loc("resource/samples") codebase = Codebase(test_codebase, strip_root=True, max_in_memory=-1) files_count, dirs_count, size_count = codebase.compute_counts() assert 33 == files_count @@ -488,35 +481,34 @@ def test_compute_counts_when_using_disk_cache(self): assert 0 == size_count def test_distance(self): - test_dir = self.get_test_loc('resource/dist') + test_dir = self.get_test_loc("resource/dist") codebase = Codebase(test_dir) assert codebase.root.distance(test_dir) == 0 - res = codebase.get_resource('dist/JGroups') - assert res.name == 'JGroups' + res = codebase.get_resource("dist/JGroups") + assert res.name == "JGroups" assert res.distance(codebase) == 1 - res = codebase.get_resource('dist/simple/META-INF/MANIFEST.MF') - assert res.name == 'MANIFEST.MF' - assert res.full_root_path.endswith( - 'resource/dist/simple/META-INF/MANIFEST.MF') + res = codebase.get_resource("dist/simple/META-INF/MANIFEST.MF") + assert res.name == "MANIFEST.MF" + assert res.full_root_path.endswith("resource/dist/simple/META-INF/MANIFEST.MF") assert res.distance(codebase) == 3 def test_skip_files_and_subdirs_of_ignored_dirs(self): - test_dir = self.get_test_loc('resource/ignore') + test_dir = self.get_test_loc("resource/ignore") codebase = Codebase(test_dir) # The `cvs` directory should not be visited - expected = ['ignore', 'ignore/file1'] + expected = ["ignore", "ignore/file1"] result = [r.path for r in codebase.walk(topdown=True)] self.assertEqual(expected, result) def test_depth_negative_fails(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") with self.assertRaises(Exception): next(depth_walk(test_codebase, -1)) def test_depth_walk_with_depth_0(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results_zero = list(depth_walk(test_codebase, 0)) results_neg = list(depth_walk(test_codebase, float("inf"))) result_zero_dirs = [i for j in results_zero for i in j[1]] @@ -527,170 +519,165 @@ def test_depth_walk_with_depth_0(self): self.assertEqual(result_neg_files, result_zero_files) def test_depth_walk_with_depth_1(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 1)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() - expected_files = ['level1_file1', 'level1_file2'].sort() - expected_dirs = ['level1_dir1', 'level1_dir2'].sort() + expected_files = ["level1_file1", "level1_file2"].sort() + expected_dirs = ["level1_dir1", "level1_dir2"].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_depth_walk_with_depth_2(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 2)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() expected_files = [ - 'level1_file1', - 'level1_file2', - 'level2_file2', - 'level2_file1', - 'level2_file3', - 'level2_file4', - 'level2_file5', + "level1_file1", + "level1_file2", + "level2_file2", + "level2_file1", + "level2_file3", + "level2_file4", + "level2_file5", ].sort() - expected_dirs = ['level1_dir1', 'level1_dir2', - 'level2_dir1', 'level2_dir3'].sort() + expected_dirs = ["level1_dir1", "level1_dir2", "level2_dir1", "level2_dir3"].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_depth_walk_with_depth_3(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 3)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() expected_files = [ - 'level1_file1', - 'level1_file2', - 'level2_file2', - 'level2_file1', - 'level3_file2', - 'level3_file1', - 'level2_file3', - 'level2_file4', - 'level2_file5', - 'level3_file4', - 'level3_file3', + "level1_file1", + "level1_file2", + "level2_file2", + "level2_file1", + "level3_file2", + "level3_file1", + "level2_file3", + "level2_file4", + "level2_file5", + "level3_file4", + "level3_file3", ].sort() expected_dirs = [ - 'level1_dir1', - 'level1_dir2', - 'level2_dir1', - 'level3_dir1', - 'level2_dir3', + "level1_dir1", + "level1_dir2", + "level2_dir1", + "level3_dir1", + "level2_dir3", ].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_specify_depth_1(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=1) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_dir1', False), - ('level1_dir2', False), - ('level1_file1', True), - ('level1_file2', True), + ("deeply_nested", False), + ("level1_dir1", False), + ("level1_dir2", False), + ("level1_file1", True), + ("level1_file2", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_specify_depth_2(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=2) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_file1', True), - ('level1_file2', True), - ('level1_dir1', False), - ('level2_dir1', False), - ('level2_file1', True), - ('level2_file2', True), - ('level1_dir2', False), - ('level2_dir3', False), - ('level2_file3', True), - ('level2_file4', True), - ('level2_file5', True), + ("deeply_nested", False), + ("level1_file1", True), + ("level1_file2", True), + ("level1_dir1", False), + ("level2_dir1", False), + ("level2_file1", True), + ("level2_file2", True), + ("level1_dir2", False), + ("level2_dir3", False), + ("level2_file3", True), + ("level2_file4", True), + ("level2_file5", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_specify_depth_3(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=3) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_file1', True), - ('level1_file2', True), - ('level1_dir1', False), - ('level2_file1', True), - ('level2_file2', True), - ('level2_dir1', False), - ('level3_dir1', False), - ('level3_file1', True), - ('level3_file2', True), - ('level1_dir2', False), - ('level2_file3', True), - ('level2_file4', True), - ('level2_file5', True), - ('level2_dir3', False), - ('level3_file3', True), - ('level3_file4', True), + ("deeply_nested", False), + ("level1_file1", True), + ("level1_file2", True), + ("level1_dir1", False), + ("level2_file1", True), + ("level2_file2", True), + ("level2_dir1", False), + ("level3_dir1", False), + ("level3_file1", True), + ("level3_file2", True), + ("level1_dir2", False), + ("level2_file3", True), + ("level2_file4", True), + ("level2_file5", True), + ("level2_dir3", False), + ("level3_file3", True), + ("level3_file4", True), ] assert [(r.name, r.is_file) for r in results] == expected class TestCodebaseWithPath(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_Codebase_with_paths_works(self): - test_codebase = self.get_test_loc('resource/with_path/codebase') - paths = ['codebase/other dir/file'] + test_codebase = self.get_test_loc("resource/with_path/codebase") + paths = ["codebase/other dir/file"] codebase = Codebase(location=test_codebase, paths=paths) assert not codebase.errors results = [r.to_dict() for r in codebase.walk()] print(r.path for r in codebase.walk()) expected_file = self.get_test_loc( - 'resource/with_path/codebase-expected.json', + "resource/with_path/codebase-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_paths_works(self): - test_codebase = self.get_test_loc( - 'resource/with_path/virtual-codebase.json') - paths = ['codebase/other dir/file'] + test_codebase = self.get_test_loc("resource/with_path/virtual-codebase.json") + paths = ["codebase/other dir/file"] codebase = VirtualCodebase(location=test_codebase, paths=paths) assert not codebase.errors results = [r.to_dict() for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/with_path/virtual-codebase-expected.json', + "resource/with_path/virtual-codebase-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_codebase_attributes_assignment(self): - test_codebase = self.get_test_loc( - 'resource/with_path/virtual-codebase.json') + test_codebase = self.get_test_loc("resource/with_path/virtual-codebase.json") vc = VirtualCodebase( location=test_codebase, - codebase_attributes=dict( - packages=attr.ib(default=attr.Factory(list)) - ), + codebase_attributes=dict(packages=attr.ib(default=attr.Factory(list))), ) self.assertNotEqual(vc.attributes.packages, None) self.assertEqual(vc.attributes.packages, []) class TestCodebaseCache(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_codebase_cache_default(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase) assert codebase.temp_dir @@ -702,15 +689,14 @@ def test_codebase_cache_default(self): assert not exists(cp) assert exists(parent_directory(cp)) - child = codebase._get_or_create_resource( - name='child', parent=root, is_file=True) + child = codebase._get_or_create_resource(name="child", parent=root, is_file=True) child.size = 12 codebase.save_resource(child) child_2 = codebase.get_resource(path=child.path) assert child_2 == child def test_codebase_cache_all_in_memory(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=0) for path, res in codebase.resources_by_path.items(): if res is Codebase.CACHED_RESOURCE: @@ -730,7 +716,7 @@ def test_codebase_cache_all_in_memory(self): ) def test_codebase_cache_all_on_disk(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=-1) for path, res in codebase.resources_by_path.items(): if res is Codebase.CACHED_RESOURCE: @@ -750,7 +736,7 @@ def test_codebase_cache_all_on_disk(self): ) def test_codebase_cache_mixed_two_in_memory(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=2) counter = 0 for path, res in codebase.resources_by_path.items(): @@ -778,115 +764,105 @@ def test_codebase_cache_mixed_two_in_memory(self): class TestVirtualCodebase(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_virtual_codebase_walk_defaults(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk()) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_topdown(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=True)) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_bottomup(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=False)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('dir', False), - ('file', True), - ('other dir', False), - ('codebase', False), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("dir", False), + ("file", True), + ("other dir", False), + ("codebase", False), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_basic(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc("resource/virtual_codebase/stripped-and-skipped-root.json") virtual_codebase = VirtualCodebase(location=scan_data) - results = [r.get_path(strip_root=True) - for r in virtual_codebase.walk(skip_root=True)] - expected = ['README', 'screenshot.png'] + results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)] + expected = ["README", "screenshot.png"] assert expected == results def test_virtual_codebase_to_list_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc("resource/virtual_codebase/stripped-and-skipped-root.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.to_list(strip_root=True, skinny=True) - expected = [{'path': 'README', 'type': 'file'}, - {'path': 'screenshot.png', 'type': 'file'}] + expected = [{"path": "README", "type": "file"}, {"path": "screenshot.png", "type": "file"}] assert expected == results def test_virtual_codebase_walk_filtered_with_filtered_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) results = list(virtual_codebase.walk_filtered()) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_all_filtered(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -897,16 +873,14 @@ def test_virtual_codebase_walk_filtered_with_all_filtered(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_compute_counts_filtered_None(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_size(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -918,16 +892,14 @@ def test_virtual_codebase_compute_counts_filtered_None_with_size(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_cache(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_all(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -937,8 +909,7 @@ def test_virtual_codebase_compute_counts_filtered_all(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -948,8 +919,7 @@ def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_files(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -960,8 +930,7 @@ def test_virtual_codebase_compute_counts_filtered_files(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_dirs(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -972,8 +941,7 @@ def test_virtual_codebase_compute_counts_filtered_dirs(self): assert results == expected def test_virtual_codebase_walk_filtered_dirs(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -981,35 +949,33 @@ def test_virtual_codebase_walk_filtered_dirs(self): virtual_codebase.save_resource(res) results = list(virtual_codebase.walk_filtered(topdown=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) results = list(virtual_codebase.walk_filtered(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_all_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -1020,24 +986,21 @@ def test_virtual_codebase_walk_filtered_all_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_not_filtered(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) @@ -1046,38 +1009,35 @@ def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file_with_children(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( - 'some child', + "some child", parent=virtual_codebase.root, is_file=True, ) _c2 = virtual_codebase._get_or_create_resource( - 'some child2', + "some child2", parent=c1, is_file=False, ) results = list(virtual_codebase.walk(skip_root=True)) - expected = [('et131x.h', True), ('some child', True), - ('some child2', False)] + expected = [("et131x.h", True), ("some child", True), ("some child2", False)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_children(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( - 'some child', + "some child", parent=virtual_codebase.root, is_file=True, ) c2 = virtual_codebase._get_or_create_resource( - 'some child2', + "some child2", parent=c1, is_file=False, ) @@ -1085,197 +1045,181 @@ def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_chil c2.save(virtual_codebase) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected c1.is_filtered = True c1.save(virtual_codebase) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( - 'some child', + "some child", virtual_codebase.root, is_file=True, ) results = list(virtual_codebase.walk()) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_dir(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc("resource/virtual_codebase/resource.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( - 'some child', + "some child", virtual_codebase.root, is_file=False, ) results = list(virtual_codebase.walk()) - expected = [('resource', False), ('some child', False)] + expected = [("resource", False), ("some child", False)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_resource(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc("resource/virtual_codebase/resource.json") virtual_codebase = VirtualCodebase(location=scan_data) - assert not ( - virtual_codebase.root is virtual_codebase.get_resource('resource')) - assert virtual_codebase.get_resource( - 'resource') == virtual_codebase.root + assert not (virtual_codebase.root is virtual_codebase.get_resource("resource")) + assert virtual_codebase.get_resource("resource") == virtual_codebase.root def test_virtual_codebase_can_process_minimal_resources_without_info(self): - scan_data = self.get_test_loc('resource/virtual_codebase/noinfo.json') + scan_data = self.get_test_loc("resource/virtual_codebase/noinfo.json") codebase = VirtualCodebase(location=scan_data) expected = [ { - 'path': 'NOTICE', - 'type': 'file', - 'copyrights': [ + "path": "NOTICE", + "type": "file", + "copyrights": [ { - 'statements': ['Copyright (c) 2017 nexB Inc. and others.'], - 'holders': ['nexB Inc. and others.'], - 'authors': [], - 'start_line': 4, - 'end_line': 4, + "statements": ["Copyright (c) 2017 nexB Inc. and others."], + "holders": ["nexB Inc. and others."], + "authors": [], + "start_line": 4, + "end_line": 4, } ], - 'scan_errors': [], + "scan_errors": [], } ] assert [r.to_dict() for r in codebase.walk()] == expected def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc("resource/virtual_codebase/only-path.json") codebase = VirtualCodebase(location=scan_data) expected = [ - {'path': 'samples', 'type': 'directory', 'scan_errors': []}, - {'path': 'samples/NOTICE', 'type': 'file', 'scan_errors': []}, + {"path": "samples", "type": "directory", "scan_errors": []}, + {"path": "samples/NOTICE", "type": "file", "scan_errors": []}, ] assert [r.to_dict() for r in codebase.walk()] == expected def test_VirtualCodebase_account_fingerprint_attribute(self): - test_file = self.get_test_loc( - "resource/virtual_codebase/fingerprint_attribute.json") + test_file = self.get_test_loc("resource/virtual_codebase/fingerprint_attribute.json") codebase = VirtualCodebase(test_file) - resources_fingerprint = [ - resource.fingerprint for resource in codebase.walk()] + resources_fingerprint = [resource.fingerprint for resource in codebase.walk()] assert "e30cf09443e7878dfed3288886e97542" in resources_fingerprint assert None in resources_fingerprint - assert codebase.get_resource( - 'apache_to_all_notable_lic_new') == codebase.root + assert codebase.get_resource("apache_to_all_notable_lic_new") == codebase.root assert resources_fingerprint.count(None) == 2 def test_VirtualCodebase_works_with_mapping_backed_codebase(self): - test_file = self.get_test_loc( - "resource/virtual_codebase/license-scan.json") + test_file = self.get_test_loc("resource/virtual_codebase/license-scan.json") codebase = VirtualCodebase(test_file) - resource = codebase.get_resource('scan-ref/license-notice.txt') + resource = codebase.get_resource("scan-ref/license-notice.txt") assert resource assert len(resource.license_expressions) == 1 class TestCodebaseLowestCommonParent(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_lowest_common_parent_on_virtual_codebase(self): - scan_data = self.get_test_loc('resource/virtual_codebase/lcp.json') + scan_data = self.get_test_loc("resource/virtual_codebase/lcp.json") virtual_codebase = VirtualCodebase(location=scan_data) lcp = virtual_codebase.lowest_common_parent() - assert lcp.path == 'lcp/test1' - assert lcp.name == 'test1' + assert lcp.path == "lcp/test1" + assert lcp.name == "test1" def test_virtual_codebase_has_default_for_plugin_attributes(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc("resource/virtual_codebase/only-path.json") VirtualCodebase(location=scan_data) def test_lowest_common_parent_strip(self): - test_codebase = self.get_test_loc('resource/lcp/test1') + test_codebase = self.get_test_loc("resource/lcp/test1") codebase = Codebase(test_codebase) assert len(list(codebase.walk())) == 75 lcp = codebase.lowest_common_parent() - assert lcp.path == 'test1' - assert lcp.name == 'test1' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1') + assert lcp.path == "test1" + assert lcp.name == "test1" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1") def test_lowest_common_parent_2(self): - test_codebase = self.get_test_loc('resource/lcp/test1/zlib') + test_codebase = self.get_test_loc("resource/lcp/test1/zlib") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'zlib' - assert lcp.name == 'zlib' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1/zlib') + assert lcp.path == "zlib" + assert lcp.name == "zlib" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1/zlib") def test_lowest_common_parent_3(self): - test_codebase = self.get_test_loc('resource/lcp/test1/simple') + test_codebase = self.get_test_loc("resource/lcp/test1/simple") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'simple' - assert lcp.name == 'simple' - assert lcp.strip_root_path == '' + assert lcp.path == "simple" + assert lcp.name == "simple" + assert lcp.strip_root_path == "" def test_lowest_common_parent_deep(self): - test_codebase = self.get_test_loc('resource/lcp/test1/simple/org') + test_codebase = self.get_test_loc("resource/lcp/test1/simple/org") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'org/jvnet/glassfish/comms/sipagent' - assert lcp.name == 'sipagent' - assert lcp.strip_root_path == 'jvnet/glassfish/comms/sipagent' + assert lcp.path == "org/jvnet/glassfish/comms/sipagent" + assert lcp.name == "sipagent" + assert lcp.strip_root_path == "jvnet/glassfish/comms/sipagent" assert lcp.full_root_path.endswith( - 'resource/lcp/test1/simple/org/jvnet/glassfish/comms/sipagent' + "resource/lcp/test1/simple/org/jvnet/glassfish/comms/sipagent" ) def test_lowest_common_parent_solo_file(self): - test_codebase = self.get_test_loc('resource/lcp/test1/screenshot.png') + test_codebase = self.get_test_loc("resource/lcp/test1/screenshot.png") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'screenshot.png' - assert lcp.name == 'screenshot.png' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1/screenshot.png') + assert lcp.path == "screenshot.png" + assert lcp.name == "screenshot.png" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1/screenshot.png") class TestVirtualCodebaseCache(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_virtual_codebase_cache_default(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data) assert virtual_codebase.temp_dir assert virtual_codebase.cache_dir virtual_codebase.cache_dir root = virtual_codebase.root - cp = virtual_codebase._get_resource_cache_location( - root.path, create_dirs=False) + cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=False) assert not exists(cp) - cp = virtual_codebase._get_resource_cache_location( - root.path, create_dirs=True) + cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=True) assert not exists(cp) assert exists(parent_directory(cp)) - child = virtual_codebase._get_or_create_resource( - 'child', root, is_file=True) + child = virtual_codebase._get_or_create_resource("child", root, is_file=True) child.size = 12 virtual_codebase.save_resource(child) child_2 = virtual_codebase.get_resource(child.path) assert child_2 == child def test_virtual_codebase_cache_all_in_memory(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=0) for path, res in virtual_codebase.resources_by_path.items(): assert res != Codebase.CACHED_RESOURCE @@ -1296,20 +1240,16 @@ def test_virtual_codebase_cache_all_in_memory(self): ) def test_virtual_codebase_cache_all_on_disk(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') - virtual_codebase = VirtualCodebase( - location=scan_data, max_in_memory=-1) + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") + virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=-1) for path, res in virtual_codebase.resources_by_path.items(): - if res != Codebase.CACHED_RESOURCE: assert res.is_root else: res = virtual_codebase.get_resource(path) if res.is_root: - assert virtual_codebase.get_resource( - path) == virtual_codebase.root + assert virtual_codebase.get_resource(path) == virtual_codebase.root assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) else: @@ -1323,8 +1263,7 @@ def test_virtual_codebase_cache_all_on_disk(self): ) def test_virtual_codebase_cache_mixed_two_in_memory(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=2) counter = 0 @@ -1334,8 +1273,7 @@ def test_virtual_codebase_cache_mixed_two_in_memory(self): if res.is_root: assert ( - virtual_codebase.get_resource( - path).to_dict() == virtual_codebase.root.to_dict() + virtual_codebase.get_resource(path).to_dict() == virtual_codebase.root.to_dict() ) assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) @@ -1358,18 +1296,17 @@ def test_virtual_codebase_cache_mixed_two_in_memory(self): class TestVirtualCodebaseCreation(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_VirtualCodebase_can_be_created_from_json_file(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/from_file.json') + test_file = self.get_test_loc("resource/virtual_codebase/from_file.json") codebase = VirtualCodebase(test_file) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_can_be_created_from_json_string(self): - test_scan = ''' + test_scan = """ { "scancode_notice": "Generated with ScanCode and provided on an ....", "scancode_version": "2.9.7.post137.2e29fe3.dirty.20181120225811", @@ -1392,10 +1329,10 @@ def test_VirtualCodebase_can_be_created_from_json_string(self): } ] } - ''' + """ codebase = VirtualCodebase(test_scan) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_can_be_created_from_dict(self): @@ -1413,94 +1350,84 @@ def test_VirtualCodebase_can_be_created_from_dict(self): codebase = VirtualCodebase(test_scan) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_create_from_scan_with_no_root_and_missing_parents(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/samples-only-findings.json') + test_file = self.get_test_loc("resource/virtual_codebase/samples-only-findings.json") result_file = self.get_test_loc( - 'resource/virtual_codebase/samples-only-findings-expected.json' + "resource/virtual_codebase/samples-only-findings-expected.json" ) codebase = VirtualCodebase(test_file) expected_scan = json.load(open(result_file)) results = sorted(r.path for r in codebase.walk()) - expected = sorted(r.get('path') for r in expected_scan['files']) + expected = sorted(r.get("path") for r in expected_scan["files"]) assert results == expected def test_VirtualCodebase_check_that_already_existing_parent_is_updated_properly(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/root-is-not-first-resource.json') + test_file = self.get_test_loc("resource/virtual_codebase/root-is-not-first-resource.json") codebase = VirtualCodebase(test_file) - results = sorted((r.to_dict() for r in codebase.walk()), - key=lambda x: tuple(x.items())) + results = sorted((r.to_dict() for r in codebase.walk()), key=lambda x: tuple(x.items())) expected = [ - {'path': 'samples', 'type': 'directory', - 'summary': ['asd'], 'scan_errors': []}, - {'path': 'samples/NOTICE', 'type': 'file', - 'summary': [], 'scan_errors': []}, + {"path": "samples", "type": "directory", "summary": ["asd"], "scan_errors": []}, + {"path": "samples/NOTICE", "type": "file", "summary": [], "scan_errors": []}, ] assert results == expected def test_VirtualCodebase_create_from_multiple_scans(self): - test_file_1 = self.get_test_loc( - 'resource/virtual_codebase/combine-1.json') - test_file_2 = self.get_test_loc( - 'resource/virtual_codebase/combine-2.json') + test_file_1 = self.get_test_loc("resource/virtual_codebase/combine-1.json") + test_file_2 = self.get_test_loc("resource/virtual_codebase/combine-2.json") vinput = (test_file_1, test_file_2) codebase = VirtualCodebase(vinput) results = [r.to_dict(with_info=False) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/combine-expected.json', + "resource/virtual_codebase/combine-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_create_from_multiple_scans_shared_directory_names(self): test_file_1 = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-1.json' + "resource/virtual_codebase/combine-shared-directory-name-1.json" ) test_file_2 = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-2.json' + "resource/virtual_codebase/combine-shared-directory-name-2.json" ) vinput = (test_file_1, test_file_2) codebase = VirtualCodebase(location=vinput) results = [r.to_dict(with_info=False) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-expected.json', + "resource/virtual_codebase/combine-shared-directory-name-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_compute_counts_with_full_root_info_one(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-one.json") codebase = VirtualCodebase(test_file) resource = [r for r in codebase.walk() if r.is_file][0] - assert resource.path == 'home/foobar/scancode-toolkit/samples/README' + assert resource.path == "home/foobar/scancode-toolkit/samples/README" files_count, dirs_count, size_count = codebase.compute_counts() assert files_count == 1 assert dirs_count == 0 assert size_count == 236 def test_VirtualCodebase_with_full_root_info_one(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-one.json") codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one-expected.json', must_exist=False + "resource/virtual_codebase/full-root-info-one-expected.json", must_exist=False ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_full_root_info_many(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-many.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-many.json") codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-many-expected.json', must_exist=False + "resource/virtual_codebase/full-root-info-many-expected.json", must_exist=False ) check_against_expected_json_file(results, expected_file, regen=False) @@ -1508,41 +1435,37 @@ def test_VirtualCodebase_can_compute_counts_with_null(self): # was failing with # size_count += child.size # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType' - test_file = self.get_test_loc( - 'resource/virtual_codebase/node-16-slim.json') + test_file = self.get_test_loc("resource/virtual_codebase/node-16-slim.json") codebase = VirtualCodebase(test_file) codebase.compute_counts() def test_VirtualCodebase_can_be_created_with_single_path(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/docker-hello-world.json') + test_file = self.get_test_loc("resource/virtual_codebase/docker-hello-world.json") VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_without_RecursionError(self): # was failing with RecursionError: maximum recursion depth exceeded - test_file = self.get_test_loc( - 'resource/virtual_codebase/zephyr-binary.json') + test_file = self.get_test_loc("resource/virtual_codebase/zephyr-binary.json") VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_with_repeated_root_directory(self): paths = [ - 'to', - 'to/to', - 'to/to/to', - 'to/to/to/to', + "to", + "to/to", + "to/to/to", + "to/to/to/to", ] - resources = [{'path': path} for path in paths] - vc = VirtualCodebase(location={'files': resources}) + resources = [{"path": path} for path in paths] + vc = VirtualCodebase(location={"files": resources}) walked_paths = [r.path for r in vc.walk()] assert paths == walked_paths class TestResource(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_Resource_extracted_to_extracted_from(self): - test_file = self.get_test_loc( - 'resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc("resource/resource/test-extracted-from-to.json") codebase = VirtualCodebase(location=test_file) results = [] for r in codebase.walk(topdown=True): @@ -1555,75 +1478,74 @@ def test_Resource_extracted_to_extracted_from(self): expected = [ ( - 'test', + "test", None, None, ), ( - 'test/c', + "test/c", None, None, ), ( - 'test/foo.tar.gz', - 'test/foo.tar.gz-extract', + "test/foo.tar.gz", + "test/foo.tar.gz-extract", None, ), ( - 'test/foo.tar.gz-extract', + "test/foo.tar.gz-extract", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo', + "test/foo.tar.gz-extract/foo", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/a', + "test/foo.tar.gz-extract/foo/a", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', - 'test/foo.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ] assert results == expected def test_virtualcode_Resource_can_walk(self): - test_file = self.get_test_loc( - 'resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc("resource/resource/test-extracted-from-to.json") codebase = VirtualCodebase(location=test_file) results = [r.path for r in codebase.walk(topdown=True)] expected = [ - 'test', - 'test/c', - 'test/foo.tar.gz', - 'test/foo.tar.gz-extract', - 'test/foo.tar.gz-extract/foo', - 'test/foo.tar.gz-extract/foo/a', - 'test/foo.tar.gz-extract/foo/bar.tar.gz', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b', + "test", + "test/c", + "test/foo.tar.gz", + "test/foo.tar.gz-extract", + "test/foo.tar.gz-extract/foo", + "test/foo.tar.gz-extract/foo/a", + "test/foo.tar.gz-extract/foo/bar.tar.gz", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b", ] assert results == expected diff --git a/tests/test_saneyaml.py b/tests/test_saneyaml.py index 2010bff..fabcf88 100644 --- a/tests/test_saneyaml.py +++ b/tests/test_saneyaml.py @@ -12,15 +12,15 @@ from commoncode.testcase import FileBasedTesting -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") class TestSaneyaml(FileBasedTesting): test_data_dir = TEST_DATA_DIR def test_load_with_and_without_tags(self): - test_file_with_tag = self.get_test_loc('saneyaml/metadata1') - test_file_without_tag = self.get_test_loc('saneyaml/metadata1.notag') - with_tags = saneyaml.load(open(test_file_with_tag, 'rb').read()) - without_tags = saneyaml.load(open(test_file_without_tag, 'rb').read()) + test_file_with_tag = self.get_test_loc("saneyaml/metadata1") + test_file_without_tag = self.get_test_loc("saneyaml/metadata1.notag") + with_tags = saneyaml.load(open(test_file_with_tag, "rb").read()) + without_tags = saneyaml.load(open(test_file_without_tag, "rb").read()) assert without_tags == with_tags diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 2eb6e55..95fcb9f 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,9 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser class BaseTests(unittest.TestCase): diff --git a/tests/test_text.py b/tests/test_text.py index 4b08bc2..e151e88 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -13,7 +13,7 @@ def test_lines(): - t = '''This problem is. + t = """This problem is. It is therefore @@ -22,69 +22,75 @@ def test_lines(): However, I have -''' +""" assert len([p[1] for p in text.lines(t)]) == 5 - expected = ['This problem is.', 'It is therefore', - 'However,we', 'without introducing ..', 'However, I have'] + expected = [ + "This problem is.", + "It is therefore", + "However,we", + "without introducing ..", + "However, I have", + ] assert [p for p in text.lines(t)] == expected def test_foldcase(): - test = ' Fold THE cases of a text to lower casM' + test = " Fold THE cases of a text to lower casM" assert text.foldcase(test) == test.lower() def test_nopunctuation(): - test = '''This problem is about sequence-bunching, %^$^%**^&*©©^(*&(*()()_+)_!@@#:><>>?/./,.,';][{}{]just''' - expected = ['This', 'problem', 'is', 'about', - 'sequence', 'bunching', 'Â', 'Â', 'just'] + test = """This problem is about sequence-bunching, %^$^%**^&*©©^(*&(*()()_+)_!@@#:><>>?/./,.,';][{}{]just""" + expected = ["This", "problem", "is", "about", "sequence", "bunching", "Â", "Â", "just"] assert text.nopunctuation(test).split() == expected - test = 'This problem is about: sequence-bunching\n\n just \n' - expected = 'This problem is about sequence bunching just ' + test = "This problem is about: sequence-bunching\n\n just \n" + expected = "This problem is about sequence bunching just " assert text.nopunctuation(test) == expected def test_unixlinesep(): t = CR + LF + LF + CR + CR + LF assert text.unixlinesep(t) == LF + LF + LF + LF - assert text.unixlinesep(t, True) == ' ' + LF + LF + LF + ' ' + LF + assert text.unixlinesep(t, True) == " " + LF + LF + LF + " " + LF def test_nolinesep(): t = CR + LF + CR + CR + CR + LF - assert text.nolinesep(t) == ' ' + assert text.nolinesep(t) == " " def test_toascii(): - acc = u"ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüýÿẞß®©œŒØøÆæ₵₡¢¢Žž" - expected = r'AAAAAACEEEEIIIINOOOOOUUUUYaaaaaaceeeeiiiinooooouuuuyyZz' + acc = "ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüýÿẞß®©œŒØøÆæ₵₡¢¢Žž" + expected = r"AAAAAACEEEEIIIINOOOOOUUUUYaaaaaaceeeeiiiinooooouuuuyyZz" assert text.toascii(acc, translit=False) == expected - expected = r'AAAAAACEEEEIIIINOOOOOOUUUUYaaaaaaceeeeiiiinoooooouuuuyySsss(r)(c)oeOEOoAEae_CL/CC/Zz' + expected = ( + r"AAAAAACEEEEIIIINOOOOOOUUUUYaaaaaaceeeeiiiinoooooouuuuyySsss(r)(c)oeOEOoAEae_CL/CC/Zz" + ) assert text.toascii(acc, translit=True) == expected def test_toascii_works_with_empty_unicode_or_bytes(): - assert text.toascii(b'', translit=False) == u'' - assert text.toascii(u'', translit=True) == u'' - assert text.toascii(b'', translit=False) == u'' - assert text.toascii(u'', translit=True) == u'' + assert text.toascii(b"", translit=False) == "" + assert text.toascii("", translit=True) == "" + assert text.toascii(b"", translit=False) == "" + assert text.toascii("", translit=True) == "" def test_python_safe_name(): s = "not `\\a /`good` -safe name ??" - assert text.python_safe_name(s) == 'not___a___good___safe_name' + assert text.python_safe_name(s) == "not___a___good___safe_name" s1 = "string1++or+" s2 = "string1 +or " assert text.python_safe_name(s2) == text.python_safe_name(s1) def test_as_unicode(): - assert text.as_unicode('') == '' - assert isinstance(text.as_unicode(b'some bytes'), str) + assert text.as_unicode("") == "" + assert isinstance(text.as_unicode(b"some bytes"), str) assert text.as_unicode(None) == None try: - text.as_unicode(['foo']) - raise Exception('Exception should have been raised') + text.as_unicode(["foo"]) + raise Exception("Exception should have been raised") except AssertionError: pass diff --git a/tests/test_timeutils.py b/tests/test_timeutils.py index 1cd58f2..568b642 100644 --- a/tests/test_timeutils.py +++ b/tests/test_timeutils.py @@ -9,29 +9,28 @@ from datetime import datetime from commoncode.testcase import FileBasedTesting +from commoncode.timeutils import UTC from commoncode.timeutils import time2tstamp from commoncode.timeutils import tstamp2time -from commoncode.timeutils import UTC class TestTimeStamp(FileBasedTesting): - def test_time2tstamp_is_path_safe_and_file_is_writable(self): ts = time2tstamp() - tf = self.get_temp_file(extension='ext', dir_name=ts, file_name=ts) - fd = open(tf, 'w') - fd.write('a') + tf = self.get_temp_file(extension="ext", dir_name=ts, file_name=ts) + fd = open(tf, "w") + fd.write("a") fd.close() def test_time2tstamp_accepts_existing_datetimes(self): ts = time2tstamp() - tf = self.get_temp_file(extension='ext', dir_name=ts, file_name=ts) - fd = open(tf, 'w') - fd.write('a') + tf = self.get_temp_file(extension="ext", dir_name=ts, file_name=ts) + fd = open(tf, "w") + fd.write("a") fd.close() def test_time2tstamp_raises_on_non_datetime(self): - self.assertRaises(AttributeError, time2tstamp, 'some') + self.assertRaises(AttributeError, time2tstamp, "some") self.assertRaises(AttributeError, time2tstamp, 1) def test_time2tstamp_tstamp2time_is_idempotent(self): @@ -42,54 +41,154 @@ def test_time2tstamp_tstamp2time_is_idempotent(self): def test_tstamp2time_format(self): import re + ts = time2tstamp() - pat = r'^20\d\d-[0-1][0-9]-[0-3]\dT[0-2]\d[0-6]\d[0-6]\d.\d\d\d\d\d\d$' + pat = r"^20\d\d-[0-1][0-9]-[0-3]\dT[0-2]\d[0-6]\d[0-6]\d.\d\d\d\d\d\d$" assert re.match(pat, ts) def test_tstamp2time(self): - dt_from_ts = tstamp2time('2010-11-12T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T131415.000016") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time2(self): - dt_from_ts = tstamp2time('20101112T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T131415.000016") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time3(self): - dt_from_ts = tstamp2time('20101112T131415.000016Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T131415.000016Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time4(self): - dt_from_ts = tstamp2time('2010-11-12T131415') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T131415") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time5(self): - dt_from_ts = tstamp2time('2010-11-12T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T13:14:15") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time6(self): - dt_from_ts = tstamp2time('20101112T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time7(self): - dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time8(self): - dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time9(self): - dt_from_ts = tstamp2time('2010-06-30T21:26:40.000Z') - assert datetime(year=2010, month=6, day=30, hour=21, minute=26, - second=40, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-06-30T21:26:40.000Z") + assert ( + datetime( + year=2010, + month=6, + day=30, + hour=21, + minute=26, + second=40, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time_raise(self): - self.assertRaises(ValueError, tstamp2time, '201011A12T13:14:15Z') + self.assertRaises(ValueError, tstamp2time, "201011A12T13:14:15Z") diff --git a/tests/test_urn.py b/tests/test_urn.py index 1a85c78..6cc6ee9 100644 --- a/tests/test_urn.py +++ b/tests/test_urn.py @@ -12,135 +12,131 @@ class URNTestCase(unittest.TestCase): - def test_encode_license(self): - u1 = urn.encode('license', key='somekey') - assert u1 == 'urn:dje:license:somekey' + u1 = urn.encode("license", key="somekey") + assert u1 == "urn:dje:license:somekey" def test_encode_owner(self): - u1 = urn.encode('owner', name='somekey') - assert u1 == 'urn:dje:owner:somekey' + u1 = urn.encode("owner", name="somekey") + assert u1 == "urn:dje:owner:somekey" def test_encode_component(self): - u1 = urn.encode('component', name='name', version='version') - assert u1 == 'urn:dje:component:name:version' + u1 = urn.encode("component", name="name", version="version") + assert u1 == "urn:dje:component:name:version" def test_encode_component_no_version(self): - u1 = urn.encode('component', name='name', version='') - assert u1 == 'urn:dje:component:name:' + u1 = urn.encode("component", name="name", version="") + assert u1 == "urn:dje:component:name:" def test_encode_license_with_extra_fields_are_ignored(self): - u1 = urn.encode('license', key='somekey', junk='somejunk') - assert u1 == 'urn:dje:license:somekey' + u1 = urn.encode("license", key="somekey", junk="somejunk") + assert u1 == "urn:dje:license:somekey" def test_encode_missing_field_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('license') + urn.encode("license") def test_encode_missing_field_component_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('component', name='this') + urn.encode("component", name="this") def test_encode_unknown_object_type_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('some', key='somekey') + urn.encode("some", key="somekey") def test_encode_component_with_spaces_are_properly_quoted(self): - u1 = urn.encode('component', name='name space', - version='version space') - assert u1 == 'urn:dje:component:name+space:version+space' + u1 = urn.encode("component", name="name space", version="version space") + assert u1 == "urn:dje:component:name+space:version+space" def test_encode_leading_and_trailing_spaces_are_trimmed_and_ignored(self): - u1 = urn.encode(' component ', name=' name space ', - version=''' version space ''') - assert u1 == 'urn:dje:component:name+space:version+space' + u1 = urn.encode(" component ", name=" name space ", version=""" version space """) + assert u1 == "urn:dje:component:name+space:version+space" def test_encode_component_with_semicolon_are_properly_quoted(self): - u1 = urn.encode('component', name='name:', version=':version') - assert u1 == 'urn:dje:component:name%3A:%3Aversion' + u1 = urn.encode("component", name="name:", version=":version") + assert u1 == "urn:dje:component:name%3A:%3Aversion" def test_encode_component_with_plus_are_properly_quoted(self): - u1 = urn.encode('component', name='name+', version='version+') - assert u1 == 'urn:dje:component:name%2B:version%2B' + u1 = urn.encode("component", name="name+", version="version+") + assert u1 == "urn:dje:component:name%2B:version%2B" def test_encode_component_with_percent_are_properly_quoted(self): - u1 = urn.encode('component', name='name%', version='version%') - assert u1 == 'urn:dje:component:name%25:version%25' + u1 = urn.encode("component", name="name%", version="version%") + assert u1 == "urn:dje:component:name%25:version%25" def test_encode_object_type_case_is_not_significant(self): - u1 = urn.encode('license', key='key') - u2 = urn.encode('lICENSe', key='key') + u1 = urn.encode("license", key="key") + u2 = urn.encode("lICENSe", key="key") assert u2 == u1 def test_decode_component(self): - u = 'urn:dje:component:name:version' - parsed = ('component', {'name': 'name', 'version': 'version'}) + u = "urn:dje:component:name:version" + parsed = ("component", {"name": "name", "version": "version"}) assert urn.decode(u) == parsed def test_decode_license(self): - u = 'urn:dje:license:lic' - parsed = ('license', {'key': 'lic'}) + u = "urn:dje:license:lic" + parsed = ("license", {"key": "lic"}) assert urn.decode(u) == parsed def test_decode_org(self): - u = 'urn:dje:owner:name' - parsed = ('owner', {'name': 'name'}) + u = "urn:dje:owner:name" + parsed = ("owner", {"name": "name"}) assert urn.decode(u) == parsed def test_decode_build_is_idempotent(self): - u1 = urn.encode('component', owner__name='org%', name='name%', - version='version%') + u1 = urn.encode("component", owner__name="org%", name="name%", version="version%") m, f = urn.decode(u1) u3 = urn.encode(m, **f) assert u3 == u1 def test_decode_raise_exception_if_incorrect_prefix(self): with self.assertRaises(urn.URNValidationError): - urn.decode('arn:dje:a:a') + urn.decode("arn:dje:a:a") def test_decode_raise_exception_if_incorrect_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:x:x:x') + urn.decode("urn:x:x:x") def test_decode_raise_exception_if_incorrect_prefix_or_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('x:x:x:x') + urn.decode("x:x:x:x") def test_decode_raise_exception_if_too_short_license(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:license') + urn.decode("urn:dje:license") def test_decode_raise_exception_if_too_short_component(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:component') + urn.decode("urn:dje:component") def test_decode_raise_exception_if_too_long(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:owner:o:n') + urn.decode("urn:dje:owner:o:n") def test_decode_raise_exception_if_too_long1(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:component:o:n:v:junk') + urn.decode("urn:dje:component:o:n:v:junk") def test_decode_raise_exception_if_too_long2(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:owner:org:junk') + urn.decode("urn:dje:owner:org:junk") def test_decode_raise_exception_if_too_long3(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:license:key:junk') + urn.decode("urn:dje:license:key:junk") def test_decode_raise_exception_if_unknown_object_type(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:marshmallows:dsds') + urn.decode("urn:dje:marshmallows:dsds") def test_decode_raise_exception_if_missing_object_type(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje::dsds') + urn.decode("urn:dje::dsds") def test_encode_decode_is_idempotent(self): - object_type = 'component' - fields = {'name': 'SIP Servlets (MSS)', 'version': 'v 1.4.0.FINAL'} - encoded = 'urn:dje:component:SIP+Servlets+%28MSS%29:v+1.4.0.FINAL' + object_type = "component" + fields = {"name": "SIP Servlets (MSS)", "version": "v 1.4.0.FINAL"} + encoded = "urn:dje:component:SIP+Servlets+%28MSS%29:v+1.4.0.FINAL" assert urn.encode(object_type, **fields) == encoded assert urn.decode(encoded) == (object_type, fields) diff --git a/tests/test_version.py b/tests/test_version.py index 9f54b84..6dafa30 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -12,112 +12,111 @@ class TestVersionHint(unittest.TestCase): - def test_version_hint(self): data = { - '/xmlgraphics/fop/source/fop-1.0-src.zip': '1.0', - '/xml/xindice/xml-xindice-1.2m1-src.zip': '1.2m1', - '/xmlgraphics/fop/binaries/fop-0.94-bin-jdk1.3.tar.gz': '0.94', - '/xmlgraphics/batik/batik-src-1.7beta1.zip': '1.7beta1', - '/xmlgraphics/batik/batik-1.7-jre13.zip': '1.7', - '/xmlbeans/source/xmlbeans-2.3.0-src.tgz': '2.3.0', - '/xml/xindice/source/xml-xindice-1.2m1-src.tar.gz': '1.2m1', - '/xml/xerces-p/binaries/XML-Xerces-2.3.0-4-win32.zip': '2.3.0-4', - '/xml/xerces-p/source/XML-Xerces-2.3.0-3.tar.gz': '2.3.0-3', - '/xml/xalan-j/source/xalan-j_2_7_0-src-2jars.tar.gz': '2_7_0', - '/xml/security/java-library/xml-security-src-1_0_5D2.zip': '1_0_5D2', - '/xml/commons/binaries/xml-commons-external-1.4.01-bin.zip': '1.4.01', - '/xml/commons/xml-commons-1.0.b2.zip': '1.0.b2', - '/xml/cocoon/3.0/cocoon-all-3.0.0-alpha-1-dist.tar.gz': '3.0.0-alpha-1', - '/xerces/j/source/Xerces-J-tools.2.10.0-xml-schema-1.1-beta.tar.gz': '2.10.0', - '/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-solaris-cc-5.10.tar.gz': '3.1.1', - '/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-windows-vc-8.0.zip': '3.1.1', - '/xerces/c/2/binaries/xerces-c_2_8_0-x86-windows-vc_7_1.zip': '2_8_0', - '/ws/woden/1.0M8/apache-woden-src-1.0M8.tar.gz': '1.0M8', - '/ws/scout/0_7rc1/source/scout-0.7rc1-src.zip': '0.7rc1', - '/ws/juddi/3_0/juddi-portal-bundle-3.0.0.rc1.zip': '3.0.0.rc1', - '/ws/juddi/3_0/juddi-portal-bundle-3.0.0.beta.zip': '3.0.0.beta', - '/ws/juddi/2_0RC7/juddi-tomcat-2.0rc7.zip': '2.0rc7', - '/ws/axis2/tools/1_4_1/axis2-wsdl2code-maven-plugin-1.4.1.jar': '1.4.1', - '/ws/axis/1_4/axis-src-1_4.zip': '1_4', - '/ws/axis-c/source/win32/axis-c-1.6b-Win32-trace-src.zip': '1.6b', - '/tuscany/java/sca/2.0-M5/apache-tuscany-sca-all-2.0-M5-src.tar.gz': '2.0-M5', - '/turbine/turbine-2.3.3-rc1/source/turbine-2.3.3-RC1-src.zip': '2.3.3-RC1', - '/tomcat/tomcat-connectors/jk/binaries/win64/jk-1.2.30/ia64/symbols-1.2.30.zip': '1.2.30', - '/tomcat/tomcat-7/v7.0.0-beta/bin/apache-tomcat-7.0.0-windows-i64.zip': '7.0.0', - '/tomcat/tomcat-4/v4.1.40/bin/apache-tomcat-4.1.40-LE-jdk14.exe': '4.1.40', - '/tapestry/tapestry-src-5.1.0.5.tar.gz': '5.1.0.5', - '/spamassassin/source/Mail-SpamAssassin-rules-3.3.0.r901671.tgz': '3.3.0.r901671', - '/spamassassin/Mail-SpamAssassin-rules-3.3.1.r923257.tgz': '3.3.1.r923257', - '/shindig/1.1-BETA5-incubating/shindig-1.1-BETA5-incubating-source.zip': '1.1-BETA5', - '/servicemix/nmr/1.0.0-m3/apache-servicemix-nmr-1.0.0-m3-src.tar.gz': '1.0.0-m3', - '/qpid/0.6/qpid-dotnet-0-10-0.6.zip': '0.6', - '/openjpa/2.0.0-beta/apache-openjpa-2.0.0-beta-binary.zip': '2.0.0-beta', - '/myfaces/source/portlet-bridge-2.0.0-alpha-2-src-all.tar.gz': '2.0.0-alpha-2', - '/myfaces/source/myfaces-extval20-2.0.3-src.tar.gz': '2.0.3', - '/harmony/milestones/6.0/debian/amd64/harmony-6.0-classlib_0.0r946981-1_amd64.deb': '6.0', - '/geronimo/eclipse/updates/plugins/org.apache.geronimo.st.v21.ui_2.1.1.jar': '2.1.1', - '/directory/studio/update/1.x/plugins/org.apache.directory.studio.aciitemeditor_1.5.2.v20091211.jar': '1.5.2.v20091211', - '/db/torque/torque-3.3/source/torque-gen-3.3-RC3-src.zip': '3.3-RC3', - '/cayenne/cayenne-3.0B1.tar.gz': '3.0B1', - '/cayenne/cayenne-3.0M4-macosx.dmg': '3.0M4', - '/xmlgraphics/batik/batik-docs-current.zip': 'current', - '/xmlgraphics/batik/batik-docs-previous.zip': 'previous', - '/poi/dev/bin/poi-bin-3.7-beta1-20100620.zip': '3.7-beta1-20100620', - '/excalibur/avalon-logkit/source/excalibur-logkit-2.0.dev-0-src.zip': '2.0.dev-0', - '/db/derby/db-derby-10.4.2.0/derby_core_plugin_10.4.2.zip': '10.4.2', - '/httpd/modpython/win/2.7.1/mp152dll.zip': '2.7.1', - '/perl/mod_perl-1.31/apaci/mod_perl.config.sh': '1.31', - '/xml/xerces-j/old_xerces2/Xerces-J-bin.2.0.0.alpha.zip': '2.0.0.alpha', - '/xml/xerces-p/archives/XML-Xerces-1.7.0_0.tar.gz': '1.7.0_0', - '/httpd/docs/tools-2004-05-04.zip': '2004-05-04', - '/ws/axis2/c/M0_5/axis2c-src-M0.5.tar.gz': 'M0.5', - '/jakarta/poi/dev/src/jakarta-poi-1.8.0-dev-src.zip': '1.8.0-dev', - '/tapestry/tapestry-4.0-beta-8.zip': '4.0-beta-8', - '/openejb/3.0-beta-1/openejb-3.0-beta-1.zip': '3.0-beta-1', - '/tapestry/tapestry-4.0-rc-1.zip': '4.0-rc-1', - '/jakarta/tapestry/source/3.0-rc-3/Tapestry-3.0-rc-3-src.zip': '3.0-rc-3', - '/jakarta/lucene/binaries/lucene-1.3-final.tar.gz': '1.3-final', - '/jakarta/tapestry/binaries/3.0-beta-1a/Tapestry-3.0-beta-1a-bin.zip': '3.0-beta-1a', - '/poi/release/bin/poi-bin-3.0-FINAL-20070503.tar.gz': '3.0-FINAL-20070503', - '/harmony/milestones/M4/apache-harmony-hdk-r603534-linux-x86-32-libstdc++v6-snapshot.tar.gz': 'r603534', - '/ant/antidote/antidote-20050330.tar.bz2': '20050330', - '/apr/not-released/apr_20020725223645.tar.gz': '20020725223645', - '/ibatis/source/ibatis.net/src-revision-709676.zip': 'revision-709676', - '/ws/axis-c/source/win32/axis-c-src-1-2-win32.zip': '1-2', - '/jakarta/slide/most-recent-2.0rc1-binaries/jakarta-slide 2.0rc1 jakarta-tomcat-4.1.30.zip': '2.0rc1', - '/httpd/modpython/win/3.0.1/python2.2.1-apache2.0.43.zip': '2.2.1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.1.0.cr1_20090319213629.jar': '2.1.0.cr1_20090319213629', - '/jakarta/poi/dev/bin/poi-2.0-pre1-20030517.jar': '2.0-pre1-20030517', - '/jakarta/poi/release/bin/jakarta-poi-1.5.0-FINAL-bin.zip': '1.5.0-FINAL', - '/jakarta/poi/release/bin/poi-bin-2.0-final-20040126.zip': '2.0-final-20040126', - '/activemq/apache-activemq/5.0.0/apache-activemq-5.0.0-sources.jar': '5.0.0', - '/turbine/turbine-2.2/source/jakarta-turbine-2.2-B1.tar.gz': '2.2-B1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.cr1.jar': '2.0.0.cr1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.final_20090108225011.jar': '2.0.0.final_20090108225011', - '/ws/axis/1_2RC3/axis-src-1_2RC3.zip': '1_2RC3', - '/commons/lang/old/v1.0-b1.1/commons-lang-1.0-b1.1.zip': '1.0-b1.1', - '/commons/net/binaries/commons-net-1.2.0-release.tar.gz': '1.2.0-release', - '/ant/ivyde/2.0.0.final/apache-ivyde-2.0.0.final-200907011148-RELEASE.tgz': '2.0.0.final-200907011148-RELEASE', - '/geronimo/eclipse/updates/plugins/org.apache.geronimo.jetty.j2ee.server.v11_1.0.0.jar': 'v11_1.0.0', - '/jakarta/cactus/binaries/jakarta-cactus-13-1.7.1-fixed.zip': '1.7.1-fixed', - '/jakarta/jakarta-turbine-maven/maven/jars/maven-1.0-b5-dev.20020731.085427.jar': '1.0-b5-dev.20020731.085427', - '/xml/xalan-j/source/xalan-j_2_5_D1-src.tar.gz': '2_5_D1', - '/ws/woden/IBuilds/I20051002_1145/woden-I20051002_1145.tar.bz2': 'I20051002_1145', - '/commons/beanutils/source/commons-beanutils-1.8.0-BETA-src.tar.gz': '1.8.0-BETA', - '/cocoon/BINARIES/cocoon-2.0.3-vm14-bin.tar.gz': '2.0.3-vm14', - '/felix/xliff_filters_v1_2_7_unix.jar': 'v1_2_7', - '/excalibur/releases/200702/excalibur-javadoc-r508111-15022007.tar.gz': 'r508111-15022007', - '/geronimo/eclipse/updates/features/org.apache.geronimo.v20.feature_2.0.0.jar': 'v20.feature_2.0.0', - '/geronimo/2.1.6/axis2-jaxws-1.3-G20090406.jar': '1.3-G20090406', - '/cassandra/debian/pool/main/c/cassandra/cassandra_0.4.0~beta1-1.diff.gz': '0.4.0~beta1-1', - '/ha-api-3.1.6.jar': '3.1.6', - 'ha-api-3.1.6.jar': '3.1.6' + "/xmlgraphics/fop/source/fop-1.0-src.zip": "1.0", + "/xml/xindice/xml-xindice-1.2m1-src.zip": "1.2m1", + "/xmlgraphics/fop/binaries/fop-0.94-bin-jdk1.3.tar.gz": "0.94", + "/xmlgraphics/batik/batik-src-1.7beta1.zip": "1.7beta1", + "/xmlgraphics/batik/batik-1.7-jre13.zip": "1.7", + "/xmlbeans/source/xmlbeans-2.3.0-src.tgz": "2.3.0", + "/xml/xindice/source/xml-xindice-1.2m1-src.tar.gz": "1.2m1", + "/xml/xerces-p/binaries/XML-Xerces-2.3.0-4-win32.zip": "2.3.0-4", + "/xml/xerces-p/source/XML-Xerces-2.3.0-3.tar.gz": "2.3.0-3", + "/xml/xalan-j/source/xalan-j_2_7_0-src-2jars.tar.gz": "2_7_0", + "/xml/security/java-library/xml-security-src-1_0_5D2.zip": "1_0_5D2", + "/xml/commons/binaries/xml-commons-external-1.4.01-bin.zip": "1.4.01", + "/xml/commons/xml-commons-1.0.b2.zip": "1.0.b2", + "/xml/cocoon/3.0/cocoon-all-3.0.0-alpha-1-dist.tar.gz": "3.0.0-alpha-1", + "/xerces/j/source/Xerces-J-tools.2.10.0-xml-schema-1.1-beta.tar.gz": "2.10.0", + "/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-solaris-cc-5.10.tar.gz": "3.1.1", + "/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-windows-vc-8.0.zip": "3.1.1", + "/xerces/c/2/binaries/xerces-c_2_8_0-x86-windows-vc_7_1.zip": "2_8_0", + "/ws/woden/1.0M8/apache-woden-src-1.0M8.tar.gz": "1.0M8", + "/ws/scout/0_7rc1/source/scout-0.7rc1-src.zip": "0.7rc1", + "/ws/juddi/3_0/juddi-portal-bundle-3.0.0.rc1.zip": "3.0.0.rc1", + "/ws/juddi/3_0/juddi-portal-bundle-3.0.0.beta.zip": "3.0.0.beta", + "/ws/juddi/2_0RC7/juddi-tomcat-2.0rc7.zip": "2.0rc7", + "/ws/axis2/tools/1_4_1/axis2-wsdl2code-maven-plugin-1.4.1.jar": "1.4.1", + "/ws/axis/1_4/axis-src-1_4.zip": "1_4", + "/ws/axis-c/source/win32/axis-c-1.6b-Win32-trace-src.zip": "1.6b", + "/tuscany/java/sca/2.0-M5/apache-tuscany-sca-all-2.0-M5-src.tar.gz": "2.0-M5", + "/turbine/turbine-2.3.3-rc1/source/turbine-2.3.3-RC1-src.zip": "2.3.3-RC1", + "/tomcat/tomcat-connectors/jk/binaries/win64/jk-1.2.30/ia64/symbols-1.2.30.zip": "1.2.30", + "/tomcat/tomcat-7/v7.0.0-beta/bin/apache-tomcat-7.0.0-windows-i64.zip": "7.0.0", + "/tomcat/tomcat-4/v4.1.40/bin/apache-tomcat-4.1.40-LE-jdk14.exe": "4.1.40", + "/tapestry/tapestry-src-5.1.0.5.tar.gz": "5.1.0.5", + "/spamassassin/source/Mail-SpamAssassin-rules-3.3.0.r901671.tgz": "3.3.0.r901671", + "/spamassassin/Mail-SpamAssassin-rules-3.3.1.r923257.tgz": "3.3.1.r923257", + "/shindig/1.1-BETA5-incubating/shindig-1.1-BETA5-incubating-source.zip": "1.1-BETA5", + "/servicemix/nmr/1.0.0-m3/apache-servicemix-nmr-1.0.0-m3-src.tar.gz": "1.0.0-m3", + "/qpid/0.6/qpid-dotnet-0-10-0.6.zip": "0.6", + "/openjpa/2.0.0-beta/apache-openjpa-2.0.0-beta-binary.zip": "2.0.0-beta", + "/myfaces/source/portlet-bridge-2.0.0-alpha-2-src-all.tar.gz": "2.0.0-alpha-2", + "/myfaces/source/myfaces-extval20-2.0.3-src.tar.gz": "2.0.3", + "/harmony/milestones/6.0/debian/amd64/harmony-6.0-classlib_0.0r946981-1_amd64.deb": "6.0", + "/geronimo/eclipse/updates/plugins/org.apache.geronimo.st.v21.ui_2.1.1.jar": "2.1.1", + "/directory/studio/update/1.x/plugins/org.apache.directory.studio.aciitemeditor_1.5.2.v20091211.jar": "1.5.2.v20091211", + "/db/torque/torque-3.3/source/torque-gen-3.3-RC3-src.zip": "3.3-RC3", + "/cayenne/cayenne-3.0B1.tar.gz": "3.0B1", + "/cayenne/cayenne-3.0M4-macosx.dmg": "3.0M4", + "/xmlgraphics/batik/batik-docs-current.zip": "current", + "/xmlgraphics/batik/batik-docs-previous.zip": "previous", + "/poi/dev/bin/poi-bin-3.7-beta1-20100620.zip": "3.7-beta1-20100620", + "/excalibur/avalon-logkit/source/excalibur-logkit-2.0.dev-0-src.zip": "2.0.dev-0", + "/db/derby/db-derby-10.4.2.0/derby_core_plugin_10.4.2.zip": "10.4.2", + "/httpd/modpython/win/2.7.1/mp152dll.zip": "2.7.1", + "/perl/mod_perl-1.31/apaci/mod_perl.config.sh": "1.31", + "/xml/xerces-j/old_xerces2/Xerces-J-bin.2.0.0.alpha.zip": "2.0.0.alpha", + "/xml/xerces-p/archives/XML-Xerces-1.7.0_0.tar.gz": "1.7.0_0", + "/httpd/docs/tools-2004-05-04.zip": "2004-05-04", + "/ws/axis2/c/M0_5/axis2c-src-M0.5.tar.gz": "M0.5", + "/jakarta/poi/dev/src/jakarta-poi-1.8.0-dev-src.zip": "1.8.0-dev", + "/tapestry/tapestry-4.0-beta-8.zip": "4.0-beta-8", + "/openejb/3.0-beta-1/openejb-3.0-beta-1.zip": "3.0-beta-1", + "/tapestry/tapestry-4.0-rc-1.zip": "4.0-rc-1", + "/jakarta/tapestry/source/3.0-rc-3/Tapestry-3.0-rc-3-src.zip": "3.0-rc-3", + "/jakarta/lucene/binaries/lucene-1.3-final.tar.gz": "1.3-final", + "/jakarta/tapestry/binaries/3.0-beta-1a/Tapestry-3.0-beta-1a-bin.zip": "3.0-beta-1a", + "/poi/release/bin/poi-bin-3.0-FINAL-20070503.tar.gz": "3.0-FINAL-20070503", + "/harmony/milestones/M4/apache-harmony-hdk-r603534-linux-x86-32-libstdc++v6-snapshot.tar.gz": "r603534", + "/ant/antidote/antidote-20050330.tar.bz2": "20050330", + "/apr/not-released/apr_20020725223645.tar.gz": "20020725223645", + "/ibatis/source/ibatis.net/src-revision-709676.zip": "revision-709676", + "/ws/axis-c/source/win32/axis-c-src-1-2-win32.zip": "1-2", + "/jakarta/slide/most-recent-2.0rc1-binaries/jakarta-slide 2.0rc1 jakarta-tomcat-4.1.30.zip": "2.0rc1", + "/httpd/modpython/win/3.0.1/python2.2.1-apache2.0.43.zip": "2.2.1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.1.0.cr1_20090319213629.jar": "2.1.0.cr1_20090319213629", + "/jakarta/poi/dev/bin/poi-2.0-pre1-20030517.jar": "2.0-pre1-20030517", + "/jakarta/poi/release/bin/jakarta-poi-1.5.0-FINAL-bin.zip": "1.5.0-FINAL", + "/jakarta/poi/release/bin/poi-bin-2.0-final-20040126.zip": "2.0-final-20040126", + "/activemq/apache-activemq/5.0.0/apache-activemq-5.0.0-sources.jar": "5.0.0", + "/turbine/turbine-2.2/source/jakarta-turbine-2.2-B1.tar.gz": "2.2-B1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.cr1.jar": "2.0.0.cr1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.final_20090108225011.jar": "2.0.0.final_20090108225011", + "/ws/axis/1_2RC3/axis-src-1_2RC3.zip": "1_2RC3", + "/commons/lang/old/v1.0-b1.1/commons-lang-1.0-b1.1.zip": "1.0-b1.1", + "/commons/net/binaries/commons-net-1.2.0-release.tar.gz": "1.2.0-release", + "/ant/ivyde/2.0.0.final/apache-ivyde-2.0.0.final-200907011148-RELEASE.tgz": "2.0.0.final-200907011148-RELEASE", + "/geronimo/eclipse/updates/plugins/org.apache.geronimo.jetty.j2ee.server.v11_1.0.0.jar": "v11_1.0.0", + "/jakarta/cactus/binaries/jakarta-cactus-13-1.7.1-fixed.zip": "1.7.1-fixed", + "/jakarta/jakarta-turbine-maven/maven/jars/maven-1.0-b5-dev.20020731.085427.jar": "1.0-b5-dev.20020731.085427", + "/xml/xalan-j/source/xalan-j_2_5_D1-src.tar.gz": "2_5_D1", + "/ws/woden/IBuilds/I20051002_1145/woden-I20051002_1145.tar.bz2": "I20051002_1145", + "/commons/beanutils/source/commons-beanutils-1.8.0-BETA-src.tar.gz": "1.8.0-BETA", + "/cocoon/BINARIES/cocoon-2.0.3-vm14-bin.tar.gz": "2.0.3-vm14", + "/felix/xliff_filters_v1_2_7_unix.jar": "v1_2_7", + "/excalibur/releases/200702/excalibur-javadoc-r508111-15022007.tar.gz": "r508111-15022007", + "/geronimo/eclipse/updates/features/org.apache.geronimo.v20.feature_2.0.0.jar": "v20.feature_2.0.0", + "/geronimo/2.1.6/axis2-jaxws-1.3-G20090406.jar": "1.3-G20090406", + "/cassandra/debian/pool/main/c/cassandra/cassandra_0.4.0~beta1-1.diff.gz": "0.4.0~beta1-1", + "/ha-api-3.1.6.jar": "3.1.6", + "ha-api-3.1.6.jar": "3.1.6", } # FIXME: generate a test function for each case for path in data: expected = data[path] - if not expected.lower().startswith('v'): - expected = 'v ' + expected + if not expected.lower().startswith("v"): + expected = "v " + expected assert version.hint(path) == expected