Add tar xattr support

Author: bozaro

pkg/private/tar/build_tar.py

@@ -14,6 +14,7 @@
 """This tool build tar files from a list of inputs."""
 
 import argparse
+import base64
 import os
 import tarfile
 import tempfile
@@ -36,27 +37,47 @@ def normpath(path):
   return os.path.normpath(path).replace(os.path.sep, '/')
 
 
+def parse_xattr(xattr_list):
+  xattrs = {}
+  if xattr_list:
+    for item in xattr_list:
+      idx = item.index("=")
+      if idx < 0:
+        raise ValueError("Unexpected xattr item format {} (want key=value)".format(item))
+      key = item[:idx]
+      raw = item[idx+1:]
+      if raw.startswith("0x"):
+        xattrs[key] = bytes.fromhex(raw[2:]).decode('latin-1')
+      elif raw.startswith('"') and raw.endswith('"') and len(raw) >= 2:
+        xattrs[key] = raw[1:-1]
+      else:
+        xattrs[key] = base64.b64decode(raw).decode('latin-1')
+  return xattrs
+
+
 class TarFile(object):
   """A class to generates a TAR file."""
 
   class DebError(Exception):
     pass
 
-  def __init__(self, output, directory, compression, compressor, default_mtime):
+  def __init__(self, output, directory, compression, compressor, default_mtime, use_xattr):
     # Directory prefix on all output paths
     d = directory.strip('/')
     self.directory = (d + '/') if d else None
     self.output = output
     self.compression = compression
     self.compressor = compressor
     self.default_mtime = default_mtime
+    self.use_xattr = use_xattr
 
   def __enter__(self):
     self.tarfile = tar_writer.TarFileWriter(
         self.output,
         self.compression,
         self.compressor,
-        default_mtime=self.default_mtime)
+        default_mtime=self.default_mtime,
+        use_xattr=self.use_xattr)
     return self
 
   def __exit__(self, t, v, traceback):
@@ -74,7 +95,7 @@ def normalize_path(self, path: str) -> str:
       dest = self.directory + dest
     return dest
 
-  def add_file(self, f, destfile, mode=None, ids=None, names=None):
+  def add_file(self, f, destfile, mode=None, ids=None, names=None, xattr=None):
     """Add a file to the tar file.
 
     Args:
@@ -85,6 +106,7 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
        ids: (uid, gid) for the file to set ownership
        names: (username, groupname) for the file to set ownership. `f` will be
          copied to `self.directory/destfile` in the layer.
+       xattr: (strings) xattr list in getfattr-like output style.
     """
     dest = self.normalize_path(destfile)
     # If mode is unspecified, derive the mode from the file's mode.
@@ -101,14 +123,16 @@ def add_file(self, f, destfile, mode=None, ids=None, names=None):
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        xattr=xattr)
 
   def add_empty_file(self,
                      destfile,
                      mode=None,
                      ids=None,
                      names=None,
-                     kind=tarfile.REGTYPE):
+                     kind=tarfile.REGTYPE,
+                     xattr=None):
     """Add a file to the tar file.
 
     Args:
@@ -118,6 +142,7 @@ def add_empty_file(self,
        names: (username, groupname) for the file to set ownership.
        kind: type of the file. tarfile.DIRTYPE for directory.  An empty file
          will be created as `destfile` in the layer.
+       xattr: (strings) xattr list in getfattr-like output style.
     """
     dest = destfile.lstrip('/')  # Remove leading slashes
     # If mode is unspecified, assume read only
@@ -136,9 +161,10 @@ def add_empty_file(self,
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        xattr=xattr)
 
-  def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
+  def add_empty_dir(self, destpath, mode=None, ids=None, names=None, xattr=None):
     """Add a directory to the tar file.
 
     Args:
@@ -147,9 +173,10 @@ def add_empty_dir(self, destpath, mode=None, ids=None, names=None):
        ids: (uid, gid) for the file to set ownership
        names: (username, groupname) for the file to set ownership.  An empty
          file will be created as `destfile` in the layer.
+       xattr: (strings) xattr list in getfattr-like output style.
     """
     self.add_empty_file(
-        destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE)
+        destpath, mode=mode, ids=ids, names=names, kind=tarfile.DIRTYPE, xattr=xattr)
 
   def add_tar(self, tar):
     """Merge a tar file into the destination tar file.
@@ -163,7 +190,7 @@ def add_tar(self, tar):
     """
     self.tarfile.add_tar(tar, numeric=True, prefix=self.directory)
 
-  def add_link(self, symlink, destination, mode=None, ids=None, names=None):
+  def add_link(self, symlink, destination, mode=None, ids=None, names=None, xattr=None):
     """Add a symbolic link pointing to `destination`.
 
     Args:
@@ -174,6 +201,7 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
       ids: (uid, gid) for the file to set ownership
       names: (username, groupname) for the file to set ownership.  An empty
         file will be created as `destfile` in the layer.
+      xattr: (strings) xattr list in getfattr-like output style.
     """
     dest = self.normalize_path(symlink)
     self.tarfile.add_file(
@@ -184,7 +212,8 @@ def add_link(self, symlink, destination, mode=None, ids=None, names=None):
         uid=ids[0],
         gid=ids[1],
         uname=names[0],
-        gname=names[1])
+        gname=names[1],
+        xattr=xattr)
 
   def add_deb(self, deb):
     """Extract a debian package in the output tar.
@@ -211,7 +240,7 @@ def add_deb(self, deb):
       self.add_tar(tmpfile[1])
       os.remove(tmpfile[1])
 
-  def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
+  def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None, xattr=None):
     """Add a tree artifact to the tar file.
 
     Args:
@@ -222,6 +251,7 @@ def add_tree(self, tree_top, destpath, mode=None, ids=None, names=None):
        ids: (uid, gid) for the file to set ownership
        names: (username, groupname) for the file to set ownership. `f` will be
          copied to `self.directory/destfile` in the layer.
+       xattr: (strings) xattr list in getfattr-like output style.
     """
     # We expect /-style paths.
     tree_top = normpath(tree_top)
@@ -374,6 +404,14 @@ def main():
       '--owner_names', action='append',
       help='Specify the owner names of individual files, e.g. '
            'path/to/file=root.root.')
+  parser.add_argument(
+      '--xattr', action='append',
+      help='Specify the xattr of all files, e.g. '
+           'security.capability=0x0100000200000001000000000000000000000000')
+  parser.add_argument(
+      '--file_xattr', action='append',
+      help='Specify the xattr of individual files, e.g. '
+           'path/to/file=security.capability=0x0100000200000001000000000000000000000000')
   parser.add_argument('--stamp_from', default='',
                       help='File to find BUILD_STAMP in')
   options = parser.parse_args()
@@ -404,6 +442,18 @@ def main():
         f = f[1:]
       names_map[f] = (user, group)
 
+  default_xattr = parse_xattr(options.xattr)
+  xattr_map = {}
+  if options.file_xattr:
+    xattr_by_file = {}
+    for file_xattr in options.file_xattr:
+      (f, xattr) = helpers.SplitNameValuePairAtSeparator(file_xattr, '=')
+      xattrs = xattr_by_file.get(f, [])
+      xattrs.append(xattr)
+      xattr_by_file[f] = xattrs
+    for f in xattr_by_file:
+      xattr_map[f] = parse_xattr(xattr_by_file[f])
+
   default_ids = options.owner.split('.', 1)
   default_ids = (int(default_ids[0]), int(default_ids[1]))
   ids_map = {}
@@ -425,7 +475,8 @@ def main():
       directory = helpers.GetFlagValue(options.directory),
       compression = options.compression,
       compressor = options.compressor,
-      default_mtime=default_mtime) as output:
+      default_mtime=default_mtime,
+      use_xattr=bool(xattr_map or default_xattr)) as output:
 
     def file_attributes(filename):
       if filename.startswith('/'):
@@ -434,6 +485,7 @@ def file_attributes(filename):
           'mode': mode_map.get(filename, default_mode),
           'ids': ids_map.get(filename, default_ids),
           'names': names_map.get(filename, default_ownername),
+          'xattr': {**xattr_map.get(filename, {}), **default_xattr}
       }
 
     if options.manifest:

pkg/private/tar/tar.bzl

@@ -181,6 +181,14 @@ def _pkg_tar_impl(ctx):
                 "--owner_names",
                 "%s=%s" % (_quote(key), ctx.attr.ownernames[key]),
             )
+    if ctx.attr.xattr:
+        for item in ctx.attr.xattr:
+            args.add("--xattr", item)
+    if ctx.attr.xattrs:
+        for file in ctx.attr.xattrs:
+            xattr = ctx.attr.xattrs[file]
+            for item in xattr:
+                args.add("--file_xattr", "%s=%s" % (_quote(file), item))
     for empty_file in ctx.attr.empty_files:
         add_empty_file(content_map, empty_file, ctx.label)
     for empty_dir in ctx.attr.empty_dirs or []:
@@ -264,6 +272,8 @@ pkg_tar_impl = rule(
         "ownername": attr.string(default = "."),
         "owners": attr.string_dict(),
         "ownernames": attr.string_dict(),
+        "xattr": attr.string_list(),
+        "xattrs": attr.string_list_dict(),
         "extension": attr.string(default = "tar"),
         "symlinks": attr.string_dict(),
         "empty_files": attr.string_list(),

pkg/private/tar/tar_writer.py

@@ -47,7 +47,8 @@ def __init__(self,
                compression='',
                compressor='',
                default_mtime=None,
-               preserve_tar_mtimes=True):
+               preserve_tar_mtimes=True,
+               use_xattr=False):
     """TarFileWriter wraps tarfile.open().
 
     Args:
@@ -60,6 +61,7 @@ def __init__(self,
       preserve_tar_mtimes: if true, keep file mtimes from input tar file.
     """
     self.preserve_mtime = preserve_tar_mtimes
+    self.use_xattr = use_xattr
     if default_mtime is None:
       self.default_mtime = 0
     elif default_mtime == 'portable':
@@ -98,7 +100,7 @@ def __init__(self,
     self.name = name
 
     self.tar = tarfile.open(name=name, mode=mode, fileobj=self.fileobj,
-                            format=tarfile.GNU_FORMAT) 
+                            format=tarfile.PAX_FORMAT if use_xattr else tarfile.GNU_FORMAT)
     self.members = set()
     self.directories = set()
     # Preseed the added directory list with things we should not add. If we
@@ -191,7 +193,8 @@ def add_file(self,
                uname='',
                gname='',
                mtime=None,
-               mode=None):
+               mode=None,
+               xattr=None):
     """Add a file to the current tar.
 
     Args:
@@ -234,6 +237,14 @@ def add_file(self,
       tarinfo.mode = mode
     if link:
       tarinfo.linkname = link
+    if xattr:
+      if not self.use_xattr:
+        raise self.Error('This tar file was created without `use_xattr` flag but try to create file with xattr: {}, {}'.
+                         format(name, xattr))
+      pax_headers = {}
+      for key in xattr:
+        pax_headers["SCHILY.xattr." + key] = xattr[key]
+      tarinfo.pax_headers = pax_headers
     if content:
       content_bytes = content.encode('utf-8')
       tarinfo.size = len(content_bytes)