From ad02117e426b838e6a407cd34e071b7b12c3de47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Wed, 9 Oct 2019 10:05:50 +0200 Subject: [PATCH 1/3] tools: port Python 3 compat patches from node-gyp to gyp Refs: https://github.com/nodejs/node-gyp/pull/1820 Refs: https://github.com/nodejs/node-gyp/pull/1843 --- tools/gyp/pylib/gyp/MSVSNew.py | 2 +- tools/gyp/pylib/gyp/common.py | 3 +++ tools/gyp/pylib/gyp/easy_xml.py | 4 +-- tools/gyp/pylib/gyp/generator/analyzer.py | 2 +- tools/gyp/pylib/gyp/generator/eclipse.py | 2 +- tools/gyp/pylib/gyp/generator/make.py | 30 +++++++++++------------ tools/gyp/pylib/gyp/generator/msvs.py | 14 +++++------ tools/gyp/pylib/gyp/input.py | 4 +++ tools/gyp/pylib/gyp/xcode_emulation.py | 2 +- tools/gyp/pylib/gyp/xcode_ninja.py | 2 +- 10 files changed, 36 insertions(+), 29 deletions(-) diff --git a/tools/gyp/pylib/gyp/MSVSNew.py b/tools/gyp/pylib/gyp/MSVSNew.py index 0ec628cc1f759d..9b64e2c1c80601 100644 --- a/tools/gyp/pylib/gyp/MSVSNew.py +++ b/tools/gyp/pylib/gyp/MSVSNew.py @@ -45,7 +45,7 @@ def MakeGuid(name, seed='msvs_new'): not change when the project for a target is rebuilt. """ # Calculate a MD5 signature for the seed and name. - d = hashlib.md5(str(seed) + str(name)).hexdigest().upper() + d = hashlib.md5((str(seed) + str(name)).encode('utf-8')).hexdigest().upper() # Convert most of the signature to GUID form (discard the rest) guid = ('{' + d[:8] + '-' + d[8:12] + '-' + d[12:16] + '-' + d[16:20] + '-' + d[20:32] + '}') diff --git a/tools/gyp/pylib/gyp/common.py b/tools/gyp/pylib/gyp/common.py index e6280d2476cf67..e5ebcd9c9f2f06 100644 --- a/tools/gyp/pylib/gyp/common.py +++ b/tools/gyp/pylib/gyp/common.py @@ -394,6 +394,9 @@ def close(self): os.unlink(self.tmp_path) raise + def write(self, s): + self.tmp_file.write(s.encode('utf-8')) + return Writer() diff --git a/tools/gyp/pylib/gyp/easy_xml.py b/tools/gyp/pylib/gyp/easy_xml.py index acccb47b75ad31..86d0ba6c0c57d4 100644 --- a/tools/gyp/pylib/gyp/easy_xml.py +++ b/tools/gyp/pylib/gyp/easy_xml.py @@ -120,7 +120,7 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False, default_encoding = locale.getdefaultlocale()[1] if default_encoding and default_encoding.upper() != encoding.upper(): - xml_string = xml_string.decode(default_encoding).encode(encoding) + xml_string = xml_string.encode(encoding) # Get the old content try: @@ -132,7 +132,7 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False, # It has changed, write it if existing != xml_string: - f = open(path, 'w') + f = open(path, 'wb') f.write(xml_string) f.close() diff --git a/tools/gyp/pylib/gyp/generator/analyzer.py b/tools/gyp/pylib/gyp/generator/analyzer.py index 88d6c72d7ada61..59d73dbedbd319 100644 --- a/tools/gyp/pylib/gyp/generator/analyzer.py +++ b/tools/gyp/pylib/gyp/generator/analyzer.py @@ -671,7 +671,7 @@ def find_matching_compile_target_names(self): assert self.is_build_impacted() # Compile targets are found by searching up from changed targets. # Reset the visited status for _GetBuildTargets. - for target in self._name_to_target.itervalues(): + for target in self._name_to_target.values(): target.visited = False supplied_targets = _LookupTargets(self._supplied_target_names_no_all(), diff --git a/tools/gyp/pylib/gyp/generator/eclipse.py b/tools/gyp/pylib/gyp/generator/eclipse.py index b7c6aa951fd131..372ceec246dedb 100644 --- a/tools/gyp/pylib/gyp/generator/eclipse.py +++ b/tools/gyp/pylib/gyp/generator/eclipse.py @@ -272,7 +272,7 @@ def WriteMacros(out, eclipse_langs, defines): out.write(' \n') for lang in eclipse_langs: out.write(' \n' % lang) - for key in sorted(defines.iterkeys()): + for key in sorted(defines): out.write(' %s%s\n' % (escape(key), escape(defines[key]))) out.write(' \n') diff --git a/tools/gyp/pylib/gyp/generator/make.py b/tools/gyp/pylib/gyp/generator/make.py index acc6813966c71e..91a119c5a57694 100644 --- a/tools/gyp/pylib/gyp/generator/make.py +++ b/tools/gyp/pylib/gyp/generator/make.py @@ -800,7 +800,7 @@ def Write(self, qualified_target, base_path, output_filename, spec, configs, gyp.xcode_emulation.MacPrefixHeader( self.xcode_settings, lambda p: Sourceify(self.Absolutify(p)), self.Pchify)) - sources = filter(Compilable, all_sources) + sources = list(filter(Compilable, all_sources)) if sources: self.WriteLn(SHARED_HEADER_SUFFIX_RULES_COMMENT1) extensions = set([os.path.splitext(s)[1] for s in sources]) @@ -953,7 +953,7 @@ def WriteActions(self, actions, extra_sources, extra_outputs, outputs = [gyp.xcode_emulation.ExpandEnvVars(o, env) for o in outputs] inputs = [gyp.xcode_emulation.ExpandEnvVars(i, env) for i in inputs] - self.WriteDoCmd(outputs, map(Sourceify, map(self.Absolutify, inputs)), + self.WriteDoCmd(outputs, [Sourceify(self.Absolutify(i)) for i in inputs], part_of_all=part_of_all, command=name) # Stuff the outputs in a variable so we can refer to them later. @@ -1002,8 +1002,8 @@ def WriteRules(self, rules, extra_sources, extra_outputs, extra_sources += outputs if int(rule.get('process_outputs_as_mac_bundle_resources', False)): extra_mac_bundle_resources += outputs - inputs = map(Sourceify, map(self.Absolutify, [rule_source] + - rule.get('inputs', []))) + inputs = [Sourceify(self.Absolutify(i)) for i + in [rule_source] + rule.get('inputs', [])] actions = ['$(call do_cmd,%s_%d)' % (name, count)] if name == 'resources_grit': @@ -1126,7 +1126,7 @@ def WriteCopies(self, copies, extra_outputs, part_of_all): path = gyp.xcode_emulation.ExpandEnvVars(path, env) self.WriteDoCmd([output], [path], 'copy', part_of_all) outputs.append(output) - self.WriteLn('%s = %s' % (variable, ' '.join(map(QuoteSpaces, outputs)))) + self.WriteLn('%s = %s' % (variable, ' '.join(QuoteSpaces(o) for o in outputs))) extra_outputs.append('$(%s)' % variable) self.WriteLn() @@ -1137,7 +1137,7 @@ def WriteMacBundleResources(self, resources, bundle_deps): for output, res in gyp.xcode_emulation.GetMacBundleResources( generator_default_variables['PRODUCT_DIR'], self.xcode_settings, - map(Sourceify, map(self.Absolutify, resources))): + [Sourceify(self.Absolutify(r)) for r in resources]): _, ext = os.path.splitext(output) if ext != '.xcassets': # Make does not supports '.xcassets' emulation. @@ -1217,11 +1217,11 @@ def WriteSources(self, configs, deps, sources, self.WriteList(cflags_objcc, 'CFLAGS_OBJCC_%s' % configname) includes = config.get('include_dirs') if includes: - includes = map(Sourceify, map(self.Absolutify, includes)) + includes = [Sourceify(self.Absolutify(i)) for i in includes] self.WriteList(includes, 'INCS_%s' % configname, prefix='-I') - compilable = filter(Compilable, sources) - objs = map(self.Objectify, map(self.Absolutify, map(Target, compilable))) + compilable = list(filter(Compilable, sources)) + objs = [self.Objectify(self.Absolutify(Target(c))) for c in compilable] self.WriteList(objs, 'OBJS') for obj in objs: @@ -1293,7 +1293,7 @@ def WriteSources(self, configs, deps, sources, # If there are any object files in our input file list, link them into our # output. - extra_link_deps += filter(Linkable, sources) + extra_link_deps += list(filter(Linkable, sources)) self.WriteLn() @@ -1543,7 +1543,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps, # Bundle dependencies. Note that the code below adds actions to this # target, so if you move these two lines, move the lines below as well. - self.WriteList(map(QuoteSpaces, bundle_deps), 'BUNDLE_DEPS') + self.WriteList([QuoteSpaces(dep) for dep in bundle_deps], 'BUNDLE_DEPS') self.WriteLn('%s: $(BUNDLE_DEPS)' % QuoteSpaces(self.output)) # After the framework is built, package it. Needs to happen before @@ -1577,7 +1577,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps, if self.type == 'executable': self.WriteLn('%s: LD_INPUTS := %s' % ( QuoteSpaces(self.output_binary), - ' '.join(map(QuoteSpaces, link_deps)))) + ' '.join(QuoteSpaces(dep) for dep in link_deps))) if self.toolset == 'host' and self.flavor == 'android': self.WriteDoCmd([self.output_binary], link_deps, 'link_host', part_of_all, postbuilds=postbuilds) @@ -1599,7 +1599,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps, elif self.type == 'shared_library': self.WriteLn('%s: LD_INPUTS := %s' % ( QuoteSpaces(self.output_binary), - ' '.join(map(QuoteSpaces, link_deps)))) + ' '.join(QuoteSpaces(dep) for dep in link_deps))) self.WriteDoCmd([self.output_binary], link_deps, 'solink', part_of_all, postbuilds=postbuilds) elif self.type == 'loadable_module': @@ -1815,7 +1815,7 @@ def WriteAndroidNdkModuleRule(self, module_name, all_sources, link_deps): default_cpp_ext = ext self.WriteLn('LOCAL_CPP_EXTENSION := ' + default_cpp_ext) - self.WriteList(map(self.Absolutify, filter(Compilable, all_sources)), + self.WriteList(list(map(self.Absolutify, filter(Compilable, all_sources))), 'LOCAL_SRC_FILES') # Filter out those which do not match prefix and suffix and produce @@ -1956,7 +1956,7 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name, "%(makefile_name)s: %(deps)s\n" "\t$(call do_cmd,regen_makefile)\n\n" % { 'makefile_name': makefile_name, - 'deps': ' '.join(map(Sourceify, build_files)), + 'deps': ' '.join(Sourceify(bf) for bf in build_files), 'cmd': gyp.common.EncodePOSIXShellList( [gyp_binary, '-fmake'] + gyp.RegenerateFlags(options) + diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index 3a9e6a7aa5cd67..1aed4ca8aa7e00 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -1778,8 +1778,8 @@ def _CollapseSingles(parent, node): # such projects up one level. if (type(node) == dict and len(node) == 1 and - node.keys()[0] == parent + '.vcproj'): - return node[node.keys()[0]] + list(node)[0] == parent + '.vcproj'): + return node[list(node)[0]] if type(node) != dict: return node for child in node: @@ -1798,8 +1798,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat): # Walk down from the top until we hit a folder that has more than one entry. # In practice, this strips the top-level "src/" dir from the hierarchy in # the solution. - while len(root) == 1 and type(root[root.keys()[0]]) == dict: - root = root[root.keys()[0]] + while len(root) == 1 and type(root[list(root)[0]]) == dict: + root = root[list(root)[0]] # Collapse singles. root = _CollapseSingles('', root) # Merge buckets until everything is a root entry. @@ -2728,7 +2728,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name): platform_name = None msvs_windows_sdk_version = None - for configuration in spec['configurations'].itervalues(): + for configuration in spec['configurations'].values(): platform_name = platform_name or _ConfigPlatform(configuration) msvs_windows_sdk_version = (msvs_windows_sdk_version or _ConfigWindowsTargetPlatformVersion(configuration, version)) @@ -3299,7 +3299,7 @@ def _GetMSBuildProjectReferences(project): ['Project', guid], ['ReferenceOutputAssembly', 'false'] ] - for config in dependency.spec.get('configurations', {}).itervalues(): + for config in dependency.spec.get('configurations', {}).values(): if config.get('msvs_use_library_dependency_inputs', 0): project_ref.append(['UseLibraryDependencyInputs', 'true']) break @@ -3368,7 +3368,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): extension_to_rule_name, _GetUniquePlatforms(spec)) missing_sources = _VerifySourcesExist(sources, project_dir) - for configuration in configurations.itervalues(): + for configuration in configurations.values(): _FinalizeMSBuildSettings(spec, configuration) # Add attributes to root element diff --git a/tools/gyp/pylib/gyp/input.py b/tools/gyp/pylib/gyp/input.py index 5d4a03020a7c23..8ea869a267ef6a 100644 --- a/tools/gyp/pylib/gyp/input.py +++ b/tools/gyp/pylib/gyp/input.py @@ -942,8 +942,12 @@ def to_utf8(s): else: replacement = variables[contents] + if isinstance(replacement, bytes) and not isinstance(replacement, str): + replacement = replacement.decode("utf-8") # done on Python 3 only if type(replacement) is list: for item in replacement: + if isinstance(item, bytes) and not isinstance(item, str): + item = item.decode("utf-8") # done on Python 3 only if not contents[-1] == '/' and type(item) not in (str, int): raise GypError('Variable ' + contents + ' must expand to a string or list of strings; ' + diff --git a/tools/gyp/pylib/gyp/xcode_emulation.py b/tools/gyp/pylib/gyp/xcode_emulation.py index 25a928b3e62df9..9a58df4782af7e 100644 --- a/tools/gyp/pylib/gyp/xcode_emulation.py +++ b/tools/gyp/pylib/gyp/xcode_emulation.py @@ -1792,7 +1792,7 @@ def _HasIOSTarget(targets): def _AddIOSDeviceConfigurations(targets): """Clone all targets and append -iphoneos to the name. Configure these targets to build for iOS devices and use correct architectures for those builds.""" - for target_dict in targets.itervalues(): + for target_dict in targets.values(): toolset = target_dict['toolset'] configs = target_dict['configurations'] for config_name, simulator_config_dict in dict(configs).items(): diff --git a/tools/gyp/pylib/gyp/xcode_ninja.py b/tools/gyp/pylib/gyp/xcode_ninja.py index 5bf6b9cfb5a0bd..2bc2143340229b 100644 --- a/tools/gyp/pylib/gyp/xcode_ninja.py +++ b/tools/gyp/pylib/gyp/xcode_ninja.py @@ -85,7 +85,7 @@ def _TargetFromSpec(old_spec, params): "%s/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)" % ninja_toplevel if 'configurations' in old_spec: - for config in old_spec['configurations'].iterkeys(): + for config in old_spec['configurations']: old_xcode_settings = \ old_spec['configurations'][config].get('xcode_settings', {}) if 'IPHONEOS_DEPLOYMENT_TARGET' in old_xcode_settings: From ca10dff0cb23342ba512ae2495291e6457a54edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Wed, 9 Oct 2019 10:36:52 +0200 Subject: [PATCH 2/3] build: always use strings for compiler version in gyp files If GYP finds a string variable that can be converted to an integer, it will do it when the variable is expanded. Use "0.0" instead of "0" to force strings and be able to use comparison operations such as `gas_version >= "2.26"` in Python 3. --- configure.py | 12 ++++++------ deps/openssl/openssl.gyp | 6 +++--- deps/openssl/openssl_common.gypi | 2 +- node.gyp | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configure.py b/configure.py index 0b6f67f68ccd75..dd93d56e552526 100755 --- a/configure.py +++ b/configure.py @@ -703,7 +703,7 @@ def get_version_helper(cc, regexp): if match: return match.group(2) else: - return '0' + return '0.0' def get_nasm_version(asm): try: @@ -714,7 +714,7 @@ def get_nasm_version(asm): warn('''No acceptable ASM compiler found! Please make sure you have installed NASM from https://www.nasm.us and refer BUILDING.md.''') - return '0' + return '0.0' match = re.match(r"NASM version ([2-9]\.[0-9][0-9]+)", to_utf8(proc.communicate()[0])) @@ -722,7 +722,7 @@ def get_nasm_version(asm): if match: return match.group(1) else: - return '0' + return '0.0' def get_llvm_version(cc): return get_version_helper( @@ -755,7 +755,7 @@ def get_gas_version(cc): return match.group(1) else: warn('Could not recognize `gas`: ' + gas_ret) - return '0' + return '0.0' # Note: Apple clang self-reports as clang 4.2.0 and gcc 4.2.1. It passes # the version check more by accident than anything else but a more rigorous @@ -766,7 +766,7 @@ def check_compiler(o): if not options.openssl_no_asm and options.dest_cpu in ('x86', 'x64'): nasm_version = get_nasm_version('nasm') o['variables']['nasm_version'] = nasm_version - if nasm_version == 0: + if nasm_version == '0.0': o['variables']['openssl_no_asm'] = 1 return @@ -785,7 +785,7 @@ def check_compiler(o): # to a version that is not completely ancient. warn('C compiler too old, need gcc 4.2 or clang 3.2 (CC=%s)' % CC) - o['variables']['llvm_version'] = get_llvm_version(CC) if is_clang else 0 + o['variables']['llvm_version'] = get_llvm_version(CC) if is_clang else '0.0' # Need xcode_version or gas_version when openssl asm files are compiled. if options.without_ssl or options.openssl_no_asm or options.shared_openssl: diff --git a/deps/openssl/openssl.gyp b/deps/openssl/openssl.gyp index 60f6ee03a7a9e4..0ca75156117da8 100644 --- a/deps/openssl/openssl.gyp +++ b/deps/openssl/openssl.gyp @@ -1,8 +1,8 @@ { 'variables': { - 'gas_version%': 0, - 'llvm_version%': 0, - 'nasm_version%': 0, + 'gas_version%': '0.0', + 'llvm_version%': '0.0', + 'nasm_version%': '0.0', }, 'targets': [ { diff --git a/deps/openssl/openssl_common.gypi b/deps/openssl/openssl_common.gypi index c4b95c20aea5d4..67640a6325eb52 100644 --- a/deps/openssl/openssl_common.gypi +++ b/deps/openssl/openssl_common.gypi @@ -64,7 +64,7 @@ 'TERMIOS', ], 'conditions': [ - [ 'llvm_version==0', { + [ 'llvm_version=="0.0"', { 'cflags': ['-Wno-old-style-declaration',], }], ], diff --git a/node.gyp b/node.gyp index ced6fbc954ce68..79969a4c834760 100644 --- a/node.gyp +++ b/node.gyp @@ -290,7 +290,7 @@ '-Wl,-bnoerrmsg', ], }], - ['(OS=="linux" or OS=="mac") and llvm_version!=0', { + ['OS in ("linux", "mac") and llvm_version != "0.0"', { 'libraries': ['-latomic'], }], ], From 83dc1d786ca837ffbd997dc7640f66fa4f9557fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C3=ABl=20Zasso?= Date: Wed, 9 Oct 2019 10:59:13 +0200 Subject: [PATCH 3/3] tools: fix GYP MSVS solution generator for Python 3 --- tools/gyp/pylib/gyp/MSVSNew.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/gyp/pylib/gyp/MSVSNew.py b/tools/gyp/pylib/gyp/MSVSNew.py index 9b64e2c1c80601..740ef2c73fa71b 100644 --- a/tools/gyp/pylib/gyp/MSVSNew.py +++ b/tools/gyp/pylib/gyp/MSVSNew.py @@ -7,6 +7,7 @@ import hashlib import os import random +from operator import attrgetter import gyp.common @@ -86,7 +87,7 @@ def __init__(self, path, name = None, entries = None, self.guid = guid # Copy passed lists (or set to empty lists) - self.entries = sorted(list(entries or [])) + self.entries = sorted(entries or [], key=attrgetter('path')) self.items = list(items or []) self.entry_type_guid = ENTRY_TYPE_GUIDS['folder'] @@ -230,7 +231,7 @@ def Write(self, writer=gyp.common.WriteOnDiff): if isinstance(e, MSVSFolder): entries_to_check += e.entries - all_entries = sorted(all_entries) + all_entries = sorted(all_entries, key=attrgetter('path')) # Open file and print header f = writer(self.path)