tools: port Python 3 compat patches from node-gyp to gyp

Refs: nodejs/node-gyp#1820
Refs: nodejs/node-gyp#1843

PR-URL: #29897
Reviewed-By: Christian Clauss <[email protected]>
Reviewed-By: Sam Roberts <[email protected]>

Author: targos

tools/gyp/pylib/gyp/MSVSNew.py

@@ -45,7 +45,7 @@ def MakeGuid(name, seed='msvs_new'):
   not change when the project for a target is rebuilt.
   """
   # Calculate a MD5 signature for the seed and name.
-  d = hashlib.md5(str(seed) + str(name)).hexdigest().upper()
+  d = hashlib.md5((str(seed) + str(name)).encode('utf-8')).hexdigest().upper()
   # Convert most of the signature to GUID form (discard the rest)
   guid = ('{' + d[:8] + '-' + d[8:12] + '-' + d[12:16] + '-' + d[16:20]
           + '-' + d[20:32] + '}')

tools/gyp/pylib/gyp/common.py

@@ -394,6 +394,9 @@ def close(self):
         os.unlink(self.tmp_path)
         raise
 
+    def write(self, s):
+      self.tmp_file.write(s.encode('utf-8'))
+
   return Writer()
 
 

tools/gyp/pylib/gyp/easy_xml.py

@@ -120,7 +120,7 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False,
 
   default_encoding = locale.getdefaultlocale()[1]
   if default_encoding and default_encoding.upper() != encoding.upper():
-    xml_string = xml_string.decode(default_encoding).encode(encoding)
+    xml_string = xml_string.encode(encoding)
 
   # Get the old content
   try:
@@ -132,7 +132,7 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False,
 
   # It has changed, write it
   if existing != xml_string:
-    f = open(path, 'w')
+    f = open(path, 'wb')
     f.write(xml_string)
     f.close()
 

tools/gyp/pylib/gyp/generator/analyzer.py

@@ -671,7 +671,7 @@ def find_matching_compile_target_names(self):
     assert self.is_build_impacted()
     # Compile targets are found by searching up from changed targets.
     # Reset the visited status for _GetBuildTargets.
-    for target in self._name_to_target.itervalues():
+    for target in self._name_to_target.values():
       target.visited = False
 
     supplied_targets = _LookupTargets(self._supplied_target_names_no_all(),

tools/gyp/pylib/gyp/generator/eclipse.py

@@ -272,7 +272,7 @@ def WriteMacros(out, eclipse_langs, defines):
   out.write('    <language name="holder for library settings"></language>\n')
   for lang in eclipse_langs:
     out.write('    <language name="%s">\n' % lang)
-    for key in sorted(defines.iterkeys()):
+    for key in sorted(defines):
       out.write('      <macro><name>%s</name><value>%s</value></macro>\n' %
                 (escape(key), escape(defines[key])))
     out.write('    </language>\n')

tools/gyp/pylib/gyp/generator/make.py

@@ -800,7 +800,7 @@ def Write(self, qualified_target, base_path, output_filename, spec, configs,
           gyp.xcode_emulation.MacPrefixHeader(
               self.xcode_settings, lambda p: Sourceify(self.Absolutify(p)),
               self.Pchify))
-      sources = filter(Compilable, all_sources)
+      sources = list(filter(Compilable, all_sources))
       if sources:
         self.WriteLn(SHARED_HEADER_SUFFIX_RULES_COMMENT1)
         extensions = set([os.path.splitext(s)[1] for s in sources])
@@ -953,7 +953,7 @@ def WriteActions(self, actions, extra_sources, extra_outputs,
       outputs = [gyp.xcode_emulation.ExpandEnvVars(o, env) for o in outputs]
       inputs = [gyp.xcode_emulation.ExpandEnvVars(i, env) for i in inputs]
 
-      self.WriteDoCmd(outputs, map(Sourceify, map(self.Absolutify, inputs)),
+      self.WriteDoCmd(outputs, [Sourceify(self.Absolutify(i)) for i in inputs],
                       part_of_all=part_of_all, command=name)
 
       # Stuff the outputs in a variable so we can refer to them later.
@@ -1002,8 +1002,8 @@ def WriteRules(self, rules, extra_sources, extra_outputs,
           extra_sources += outputs
         if int(rule.get('process_outputs_as_mac_bundle_resources', False)):
           extra_mac_bundle_resources += outputs
-        inputs = map(Sourceify, map(self.Absolutify, [rule_source] +
-                                    rule.get('inputs', [])))
+        inputs = [Sourceify(self.Absolutify(i)) for i
+                  in [rule_source] + rule.get('inputs', [])]
         actions = ['$(call do_cmd,%s_%d)' % (name, count)]
 
         if name == 'resources_grit':
@@ -1126,7 +1126,7 @@ def WriteCopies(self, copies, extra_outputs, part_of_all):
         path = gyp.xcode_emulation.ExpandEnvVars(path, env)
         self.WriteDoCmd([output], [path], 'copy', part_of_all)
         outputs.append(output)
-    self.WriteLn('%s = %s' % (variable, ' '.join(map(QuoteSpaces, outputs))))
+    self.WriteLn('%s = %s' % (variable, ' '.join(QuoteSpaces(o) for o in outputs)))
     extra_outputs.append('$(%s)' % variable)
     self.WriteLn()
 
@@ -1137,7 +1137,7 @@ def WriteMacBundleResources(self, resources, bundle_deps):
 
     for output, res in gyp.xcode_emulation.GetMacBundleResources(
         generator_default_variables['PRODUCT_DIR'], self.xcode_settings,
-        map(Sourceify, map(self.Absolutify, resources))):
+        [Sourceify(self.Absolutify(r)) for r in resources]):
       _, ext = os.path.splitext(output)
       if ext != '.xcassets':
         # Make does not supports '.xcassets' emulation.
@@ -1217,11 +1217,11 @@ def WriteSources(self, configs, deps, sources,
         self.WriteList(cflags_objcc, 'CFLAGS_OBJCC_%s' % configname)
       includes = config.get('include_dirs')
       if includes:
-        includes = map(Sourceify, map(self.Absolutify, includes))
+        includes = [Sourceify(self.Absolutify(i)) for i in includes]
       self.WriteList(includes, 'INCS_%s' % configname, prefix='-I')
 
-    compilable = filter(Compilable, sources)
-    objs = map(self.Objectify, map(self.Absolutify, map(Target, compilable)))
+    compilable = list(filter(Compilable, sources))
+    objs = [self.Objectify(self.Absolutify(Target(c))) for c in compilable]
     self.WriteList(objs, 'OBJS')
 
     for obj in objs:
@@ -1293,7 +1293,7 @@ def WriteSources(self, configs, deps, sources,
 
     # If there are any object files in our input file list, link them into our
     # output.
-    extra_link_deps += filter(Linkable, sources)
+    extra_link_deps += list(filter(Linkable, sources))
 
     self.WriteLn()
 
@@ -1543,7 +1543,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps,
 
       # Bundle dependencies. Note that the code below adds actions to this
       # target, so if you move these two lines, move the lines below as well.
-      self.WriteList(map(QuoteSpaces, bundle_deps), 'BUNDLE_DEPS')
+      self.WriteList([QuoteSpaces(dep) for dep in bundle_deps], 'BUNDLE_DEPS')
       self.WriteLn('%s: $(BUNDLE_DEPS)' % QuoteSpaces(self.output))
 
       # After the framework is built, package it. Needs to happen before
@@ -1577,7 +1577,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps,
     if self.type == 'executable':
       self.WriteLn('%s: LD_INPUTS := %s' % (
           QuoteSpaces(self.output_binary),
-          ' '.join(map(QuoteSpaces, link_deps))))
+          ' '.join(QuoteSpaces(dep) for dep in link_deps)))
       if self.toolset == 'host' and self.flavor == 'android':
         self.WriteDoCmd([self.output_binary], link_deps, 'link_host',
                         part_of_all, postbuilds=postbuilds)
@@ -1599,7 +1599,7 @@ def WriteTarget(self, spec, configs, deps, link_deps, bundle_deps,
     elif self.type == 'shared_library':
       self.WriteLn('%s: LD_INPUTS := %s' % (
             QuoteSpaces(self.output_binary),
-            ' '.join(map(QuoteSpaces, link_deps))))
+            ' '.join(QuoteSpaces(dep) for dep in link_deps)))
       self.WriteDoCmd([self.output_binary], link_deps, 'solink', part_of_all,
                       postbuilds=postbuilds)
     elif self.type == 'loadable_module':
@@ -1815,7 +1815,7 @@ def WriteAndroidNdkModuleRule(self, module_name, all_sources, link_deps):
           default_cpp_ext = ext
     self.WriteLn('LOCAL_CPP_EXTENSION := ' + default_cpp_ext)
 
-    self.WriteList(map(self.Absolutify, filter(Compilable, all_sources)),
+    self.WriteList(list(map(self.Absolutify, filter(Compilable, all_sources))),
                    'LOCAL_SRC_FILES')
 
     # Filter out those which do not match prefix and suffix and produce
@@ -1956,7 +1956,7 @@ def WriteAutoRegenerationRule(params, root_makefile, makefile_name,
       "%(makefile_name)s: %(deps)s\n"
       "\t$(call do_cmd,regen_makefile)\n\n" % {
           'makefile_name': makefile_name,
-          'deps': ' '.join(map(Sourceify, build_files)),
+          'deps': ' '.join(Sourceify(bf) for bf in build_files),
           'cmd': gyp.common.EncodePOSIXShellList(
                      [gyp_binary, '-fmake'] +
                      gyp.RegenerateFlags(options) +

tools/gyp/pylib/gyp/generator/msvs.py

@@ -1778,8 +1778,8 @@ def _CollapseSingles(parent, node):
   # such projects up one level.
   if (type(node) == dict and
       len(node) == 1 and
-      node.keys()[0] == parent + '.vcproj'):
-    return node[node.keys()[0]]
+      list(node)[0] == parent + '.vcproj'):
+    return node[list(node)[0]]
   if type(node) != dict:
     return node
   for child in node:
@@ -1798,8 +1798,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
   # Walk down from the top until we hit a folder that has more than one entry.
   # In practice, this strips the top-level "src/" dir from the hierarchy in
   # the solution.
-  while len(root) == 1 and type(root[root.keys()[0]]) == dict:
-    root = root[root.keys()[0]]
+  while len(root) == 1 and type(root[list(root)[0]]) == dict:
+    root = root[list(root)[0]]
   # Collapse singles.
   root = _CollapseSingles('', root)
   # Merge buckets until everything is a root entry.
@@ -2728,7 +2728,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name):
 
   platform_name = None
   msvs_windows_sdk_version = None
-  for configuration in spec['configurations'].itervalues():
+  for configuration in spec['configurations'].values():
     platform_name = platform_name or _ConfigPlatform(configuration)
     msvs_windows_sdk_version = (msvs_windows_sdk_version or
                   _ConfigWindowsTargetPlatformVersion(configuration, version))
@@ -3299,7 +3299,7 @@ def _GetMSBuildProjectReferences(project):
           ['Project', guid],
           ['ReferenceOutputAssembly', 'false']
           ]
-      for config in dependency.spec.get('configurations', {}).itervalues():
+      for config in dependency.spec.get('configurations', {}).values():
         if config.get('msvs_use_library_dependency_inputs', 0):
           project_ref.append(['UseLibraryDependencyInputs', 'true'])
           break
@@ -3368,7 +3368,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
                               extension_to_rule_name, _GetUniquePlatforms(spec))
   missing_sources = _VerifySourcesExist(sources, project_dir)
 
-  for configuration in configurations.itervalues():
+  for configuration in configurations.values():
     _FinalizeMSBuildSettings(spec, configuration)
 
   # Add attributes to root element

tools/gyp/pylib/gyp/input.py

@@ -942,8 +942,12 @@ def to_utf8(s):
       else:
         replacement = variables[contents]
 
+    if isinstance(replacement, bytes) and not isinstance(replacement, str):
+      replacement = replacement.decode("utf-8")  # done on Python 3 only
     if type(replacement) is list:
       for item in replacement:
+        if isinstance(item, bytes) and not isinstance(item, str):
+          item = item.decode("utf-8")  # done on Python 3 only
         if not contents[-1] == '/' and type(item) not in (str, int):
           raise GypError('Variable ' + contents +
                          ' must expand to a string or list of strings; ' +

tools/gyp/pylib/gyp/xcode_emulation.py

@@ -1792,7 +1792,7 @@ def _HasIOSTarget(targets):
 def _AddIOSDeviceConfigurations(targets):
   """Clone all targets and append -iphoneos to the name. Configure these targets
   to build for iOS devices and use correct architectures for those builds."""
-  for target_dict in targets.itervalues():
+  for target_dict in targets.values():
     toolset = target_dict['toolset']
     configs = target_dict['configurations']
     for config_name, simulator_config_dict in dict(configs).items():

tools/gyp/pylib/gyp/xcode_ninja.py

@@ -85,7 +85,7 @@ def _TargetFromSpec(old_spec, params):
         "%s/$(CONFIGURATION)$(EFFECTIVE_PLATFORM_NAME)" % ninja_toplevel
 
   if 'configurations' in old_spec:
-    for config in old_spec['configurations'].iterkeys():
+    for config in old_spec['configurations']:
       old_xcode_settings = \
         old_spec['configurations'][config].get('xcode_settings', {})
       if 'IPHONEOS_DEPLOYMENT_TARGET' in old_xcode_settings: