macOS: Refactor create_macos_framework.py (#54557)

This is a refactoring with no semantic changes.

This refactors the macOS framework creation code to be more readable, and extracts it to `sky_utils.py`.

While I was pulling this out, also generalised the code to not hardcode `FlutterMacOS.framework` in case we one day manage to generate the iOS and macOS frameworks with the same name.

This is a reland of #54546 (reverted in #54549), the original was reverted in order to revert #54543 (reverted in #54550), which was reverted because it failed to preserve symlinks while zipping the macOS framework. That patch has been relanded with a fix in #54555. This patch has been rebased to tip-of-tree for attempt two.

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style

Author: cbracken

sky/tools/create_macos_framework.py

@@ -5,7 +5,6 @@
 # found in the LICENSE file.
 
 import argparse
-import subprocess
 import shutil
 import sys
 import os
@@ -38,57 +37,32 @@ def main():
       if os.path.isabs(args.x64_out_dir) else sky_utils.buildroot_relative_path(args.x64_out_dir)
   )
 
-  fat_framework_bundle = os.path.join(dst, 'FlutterMacOS.framework')
   arm64_framework = os.path.join(arm64_out_dir, 'FlutterMacOS.framework')
-  x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework')
-
-  arm64_dylib = os.path.join(arm64_framework, 'FlutterMacOS')
-  x64_dylib = os.path.join(x64_framework, 'FlutterMacOS')
-
   if not os.path.isdir(arm64_framework):
     print('Cannot find macOS arm64 Framework at %s' % arm64_framework)
     return 1
 
+  x64_framework = os.path.join(x64_out_dir, 'FlutterMacOS.framework')
   if not os.path.isdir(x64_framework):
     print('Cannot find macOS x64 Framework at %s' % x64_framework)
     return 1
 
+  arm64_dylib = sky_utils.get_framework_dylib_path(arm64_framework)
   if not os.path.isfile(arm64_dylib):
     print('Cannot find macOS arm64 dylib at %s' % arm64_dylib)
     return 1
 
+  x64_dylib = sky_utils.get_framework_dylib_path(x64_framework)
   if not os.path.isfile(x64_dylib):
     print('Cannot find macOS x64 dylib at %s' % x64_dylib)
     return 1
 
-  sky_utils.copy_tree(arm64_framework, fat_framework_bundle, symlinks=True)
-
-  regenerate_symlinks(fat_framework_bundle)
-
-  fat_framework_binary = os.path.join(fat_framework_bundle, 'Versions', 'A', 'FlutterMacOS')
-
-  # Create the arm64/x64 fat framework.
-  sky_utils.lipo([arm64_dylib, x64_dylib], fat_framework_binary)
-
-  # Make the framework readable and executable: u=rwx,go=rx.
-  subprocess.check_call(['chmod', '755', fat_framework_bundle])
-
-  # Add group and other readability to all files.
-  versions_path = os.path.join(fat_framework_bundle, 'Versions')
-  subprocess.check_call(['chmod', '-R', 'og+r', versions_path])
-  # Find all the files below the target dir with owner execute permission
-  find_subprocess = subprocess.Popen(['find', versions_path, '-perm', '-100', '-print0'],
-                                     stdout=subprocess.PIPE)
-  # Add execute permission for other and group for all files that had it for owner.
-  xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'],
-                                      stdin=find_subprocess.stdout)
-  find_subprocess.wait()
-  xargs_subprocess.wait()
-
-  process_framework(dst, args, fat_framework_bundle, fat_framework_binary)
+  fat_framework = os.path.join(dst, 'FlutterMacOS.framework')
+  sky_utils.create_fat_macos_framework(fat_framework, arm64_framework, x64_framework)
+  process_framework(dst, args, fat_framework)
 
   # Create XCFramework from the arm64 and x64 fat framework.
-  xcframeworks = [fat_framework_bundle]
+  xcframeworks = [fat_framework]
   create_xcframework(location=dst, name='FlutterMacOS', frameworks=xcframeworks)
 
   if args.zip:
@@ -97,56 +71,26 @@ def main():
   return 0
 
 
-def regenerate_symlinks(fat_framework_bundle):
-  """Regenerates the symlinks structure.
-
-  Recipes V2 upload artifacts in CAS before integration and CAS follows symlinks.
-  This logic regenerates the symlinks in the expected structure.
-  """
-  if os.path.islink(os.path.join(fat_framework_bundle, 'FlutterMacOS')):
-    return
-  os.remove(os.path.join(fat_framework_bundle, 'FlutterMacOS'))
-  shutil.rmtree(os.path.join(fat_framework_bundle, 'Headers'), True)
-  shutil.rmtree(os.path.join(fat_framework_bundle, 'Modules'), True)
-  shutil.rmtree(os.path.join(fat_framework_bundle, 'Resources'), True)
-  current_version_path = os.path.join(fat_framework_bundle, 'Versions', 'Current')
-  shutil.rmtree(current_version_path, True)
-  os.symlink('A', current_version_path)
-  os.symlink(
-      os.path.join('Versions', 'Current', 'FlutterMacOS'),
-      os.path.join(fat_framework_bundle, 'FlutterMacOS')
-  )
-  os.symlink(
-      os.path.join('Versions', 'Current', 'Headers'), os.path.join(fat_framework_bundle, 'Headers')
-  )
-  os.symlink(
-      os.path.join('Versions', 'Current', 'Modules'), os.path.join(fat_framework_bundle, 'Modules')
-  )
-  os.symlink(
-      os.path.join('Versions', 'Current', 'Resources'),
-      os.path.join(fat_framework_bundle, 'Resources')
-  )
-
+def process_framework(dst, args, framework_path):
+  framework_binary = sky_utils.get_framework_dylib_path(framework_path)
 
-def process_framework(dst, args, fat_framework_bundle, fat_framework_binary):
   if args.dsym:
-    dsym_out = os.path.splitext(fat_framework_bundle)[0] + '.dSYM'
-    sky_utils.extract_dsym(fat_framework_binary, dsym_out)
+    dsym_out = os.path.join(dst, 'FlutterMacOS.dSYM')
+    sky_utils.extract_dsym(framework_binary, dsym_out)
     if args.zip:
-      dsym_dst = os.path.join(dst, 'FlutterMacOS.dSYM')
-      sky_utils.create_zip(dsym_dst, 'FlutterMacOS.dSYM.zip', ['.'])
-      # Double zip to make it consistent with legacy artifacts.
+      # Create a zip of just the contents of the dSYM, then create a zip of that zip.
       # TODO(fujino): remove this once https://github.com/flutter/flutter/issues/125067 is resolved
-      sky_utils.create_zip(dsym_dst, 'FlutterMacOS.dSYM_.zip', ['FlutterMacOS.dSYM.zip'])
+      sky_utils.create_zip(dsym_out, 'FlutterMacOS.dSYM.zip', ['.'])
+      sky_utils.create_zip(dsym_out, 'FlutterMacOS.dSYM_.zip', ['FlutterMacOS.dSYM.zip'])
 
       # Overwrite the FlutterMacOS.dSYM.zip with the double-zipped archive.
-      dsym_final_src_path = os.path.join(dsym_dst, 'FlutterMacOS.dSYM_.zip')
+      dsym_final_src_path = os.path.join(dsym_out, 'FlutterMacOS.dSYM_.zip')
       dsym_final_dst_path = os.path.join(dst, 'FlutterMacOS.dSYM.zip')
       shutil.move(dsym_final_src_path, dsym_final_dst_path)
 
   if args.strip:
     unstripped_out = os.path.join(dst, 'FlutterMacOS.unstripped')
-    sky_utils.strip_binary(fat_framework_binary, unstripped_out)
+    sky_utils.strip_binary(framework_binary, unstripped_out)
 
 
 def zip_framework(dst):

sky/tools/sky_utils.py

@@ -54,13 +54,84 @@ def create_zip(cwd, zip_filename, paths):
   subprocess.check_call(['zip', '-r', '-y', zip_filename] + paths, cwd=cwd)
 
 
+def create_fat_macos_framework(fat_framework, arm64_framework, x64_framework):
+  """Creates a fat framework from two arm64 and x64 frameworks."""
+  # Clone the arm64 framework bundle as a starting point.
+  copy_tree(arm64_framework, fat_framework, symlinks=True)
+  _regenerate_symlinks(fat_framework)
+  lipo([get_framework_dylib_path(arm64_framework),
+        get_framework_dylib_path(x64_framework)], get_framework_dylib_path(fat_framework))
+  _set_framework_permissions(fat_framework)
+
+
+def _regenerate_symlinks(framework_path):
+  """Regenerates the framework symlink structure.
+
+  When building on the bots, the framework is produced in one shard, uploaded
+  to LUCI's content-addressable storage cache (CAS), then pulled down in
+  another shard. When that happens, symlinks are dereferenced, resulting a
+  corrupted framework. This regenerates the expected symlink farm.
+  """
+  # If the dylib is symlinked, assume symlinks are all fine and bail out.
+  # The shutil.rmtree calls below only work on directories, and fail on symlinks.
+  framework_name = get_framework_name(framework_path)
+  framework_binary = get_framework_dylib_path(framework_path)
+  if os.path.islink(os.path.join(framework_path, framework_name)):
+    return
+
+  # Delete any existing files/directories.
+  os.remove(framework_binary)
+  shutil.rmtree(os.path.join(framework_path, 'Headers'), True)
+  shutil.rmtree(os.path.join(framework_path, 'Modules'), True)
+  shutil.rmtree(os.path.join(framework_path, 'Resources'), True)
+  current_version_path = os.path.join(framework_path, 'Versions', 'Current')
+  shutil.rmtree(current_version_path, True)
+
+  # Recreate the expected framework symlinks.
+  os.symlink('A', current_version_path)
+  os.symlink(os.path.join(current_version_path, framework_name), framework_binary)
+  os.symlink(os.path.join(current_version_path, 'Headers'), os.path.join(framework_path, 'Headers'))
+  os.symlink(os.path.join(current_version_path, 'Modules'), os.path.join(framework_path, 'Modules'))
+  os.symlink(
+      os.path.join(current_version_path, 'Resources'), os.path.join(framework_path, 'Resources')
+  )
+
+
+def _set_framework_permissions(framework_dir):
+  """Sets framework contents to be world readable, and world executable if user-executable."""
+  # Make the framework readable and executable: u=rwx,go=rx.
+  subprocess.check_call(['chmod', '755', framework_dir])
+
+  # Add group and other readability to all files.
+  subprocess.check_call(['chmod', '-R', 'og+r', framework_dir])
+
+  # Find all the files below the target dir with owner execute permission and
+  # set og+x where it had the execute permission set for the owner.
+  find_subprocess = subprocess.Popen(['find', framework_dir, '-perm', '-100', '-print0'],
+                                     stdout=subprocess.PIPE)
+  xargs_subprocess = subprocess.Popen(['xargs', '-0', 'chmod', 'og+x'],
+                                      stdin=find_subprocess.stdout)
+  find_subprocess.wait()
+  xargs_subprocess.wait()
+
+
 def _dsymutil_path():
   """Returns the path to dsymutil within Flutter's clang toolchain."""
   arch_subpath = 'mac-arm64' if platform.processor() == 'arm' else 'mac-x64'
   dsymutil_path = os.path.join('flutter', 'buildtools', arch_subpath, 'clang', 'bin', 'dsymutil')
   return buildroot_relative_path(dsymutil_path)
 
 
+def get_framework_name(framework_dir):
+  """Returns Foo given /path/to/Foo.framework."""
+  return os.path.splitext(os.path.basename(framework_dir))[0]
+
+
+def get_framework_dylib_path(framework_dir):
+  """Returns /path/to/Foo.framework/Versions/A/Foo given /path/to/Foo.framework."""
+  return os.path.join(framework_dir, 'Versions', 'A', get_framework_name(framework_dir))
+
+
 def extract_dsym(binary_path, dsym_out_path):
   """Extracts a dSYM bundle from the specified Mach-O binary."""
   arch_dir = 'mac-arm64' if platform.processor() == 'arm' else 'mac-x64'