Syncing recent changes. (#1099)

* Added support for lising %SystemDrive%\Users
* Removed support for the GREP artifact source.
* Removed now unused parsers code.
* Removed the accidentally introduced dependency of grr-api-client on
  grr-response-core.
* Migrated more API handlers to not use RDFProtoStructs.
* Adjusted the file formatting.
* Bumped the version to 3.4.7.5.

Author: mbushkov

CHANGELOG.md

@@ -7,10 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Added support for listing `%SystemDrive%\Users` as a supplementary mechanism
+  for collecting user profiles on Windows (additionally to using data from the
+  registry).
+
 ### Removed
 
 * Removed the `ListFlowApplicableParsers` API method.
 * Removed the `ListParsedFlowResults` API method.
+* Removed support for the `GREP` artifact source (these were internal to GRR and
+  not part of the [official specification](https://artifacts.readthedocs.io/en/latest/sources/Format-specification.html).
 
 ## [3.4.7.4] - 2024-05-28
 

Dockerfile

@@ -1,6 +1,6 @@
 # A Docker image capable of running all GRR components.
 #
-# See https://hub.docker.com/r/grrdocker/grr/
+# See https://github.com/google/grr/pkgs/container/grr
 #
 # We have configured Github Actions to trigger an image build every
 # time a new a PUSH happens in the GRR github repository.

api_client/python/grr_api_client/client.py

@@ -2,12 +2,12 @@
 """Clients-related part of GRR API client library."""
 
 from collections import abc
+import time
 from typing import Sequence
 
 from grr_api_client import flow
 from grr_api_client import utils
 from grr_api_client import vfs
-from grr_response_core.lib import rdfvalue
 from grr_response_proto.api import client_pb2
 from grr_response_proto.api import flow_pb2
 from grr_response_proto.api import user_pb2
@@ -209,10 +209,9 @@ def CreateApproval(
 
     expiration_time_us = 0
     if expiration_duration_days != 0:
-      expiration_time_us = (
-          rdfvalue.RDFDatetime.Now()
-          + rdfvalue.Duration.From(expiration_duration_days, rdfvalue.DAYS)
-      ).AsMicrosecondsSinceEpoch()
+      expiration_time_us = int(
+          (time.time() + expiration_duration_days * 24 * 3600) * 1e6
+      )
 
     approval = user_pb2.ApiClientApproval(
         reason=reason,

api_client/python/grr_api_client/flow.py

@@ -11,7 +11,6 @@
 from grr_api_client import context as api_context
 from grr_api_client import errors
 from grr_api_client import utils
-from grr_response_core.lib.util import aead
 from grr_response_proto.api import flow_pb2
 from grr_response_proto.api import osquery_pb2
 from grr_response_proto.api import timeline_pb2
@@ -268,5 +267,5 @@ def DecryptLargeFile(
 
     with input_context as input_stream:
       with output_context as output_stream:
-        decrypted_stream = aead.Decrypt(input_stream, encryption_key)
+        decrypted_stream = utils.AEADDecrypt(input_stream, encryption_key)
         shutil.copyfileobj(decrypted_stream, output_stream)

api_client/python/grr_api_client/utils.py

@@ -1,6 +1,9 @@
 #!/usr/bin/env python
 """Utility functions and classes for GRR API client library."""
 
+import io
+import itertools
+import struct
 import time
 from typing import Any
 from typing import Callable
@@ -11,6 +14,8 @@
 from typing import TypeVar
 from typing import Union
 
+from cryptography.hazmat.primitives.ciphers import aead
+
 from google.protobuf import any_pb2
 from google.protobuf import wrappers_pb2
 from google.protobuf import descriptor
@@ -307,6 +312,97 @@ def Xor(bytestr: bytes, key: int) -> bytes:
   return bytes([byte ^ key for byte in bytestr])
 
 
+class _Unchunked(io.RawIOBase, IO[bytes]):  # pytype: disable=signature-mismatch  # overriding-return-type-checks
+  """A raw file-like object that reads chunk stream on demand."""
+
+  def __init__(self, chunks: Iterator[bytes]) -> None:
+    """Initializes the object."""
+    super().__init__()
+    self._chunks = chunks
+    self._buf = io.BytesIO()
+
+  def readable(self) -> bool:
+    return True
+
+  def readall(self) -> bytes:
+    return b"".join(self._chunks)
+
+  def readinto(self, buf: bytearray) -> int:
+    if self._buf.tell() == len(self._buf.getbuffer()):
+      self._buf.seek(0, io.SEEK_SET)
+      self._buf.truncate()
+      self._buf.write(next(self._chunks, b""))
+      self._buf.seek(0, io.SEEK_SET)
+
+    return self._buf.readinto(buf)
+
+
+def AEADDecrypt(stream: IO[bytes], key: bytes) -> IO[bytes]:
+  """Decrypts given file-like object using AES algorithm in GCM mode.
+
+  Refer to the encryption documentation to learn about the details of the format
+  that this function allows to decode.
+
+  Args:
+    stream: A file-like object to decrypt.
+    key: A secret key used for decrypting the data.
+
+  Returns:
+    A file-like object with decrypted data.
+  """
+  aesgcm = aead.AESGCM(key)
+
+  def Generate() -> Iterator[bytes]:
+    # Buffered reader should accept `IO[bytes]` but for now it accepts only
+    # `RawIOBase` (which is a concrete base class for all I/O implementations).
+    reader = io.BufferedReader(stream)  # pytype: disable=wrong-arg-types
+
+    # We abort early if there is no data in the stream. Otherwise we would try
+    # to read nonce and fail.
+    if not reader.peek():
+      return
+
+    for idx in itertools.count():
+      nonce = reader.read(_AEAD_NONCE_SIZE)
+
+      # As long there is some data in the buffer (and there should be because of
+      # the initial check) there should be a fixed-size nonce prepended to each
+      # chunk.
+      if len(nonce) != _AEAD_NONCE_SIZE:
+        raise EOFError(f"Incorrect nonce length: {len(nonce)}")
+
+      chunk = reader.read(_AEAD_CHUNK_SIZE + 16)
+
+      # `BufferedReader#peek` will return non-empty byte string if there is more
+      # data available in the stream.
+      is_last = reader.peek() == b""  # pylint: disable=g-explicit-bool-comparison
+
+      adata = _AEAD_ADATA_FORMAT.pack(idx, is_last)
+
+      yield aesgcm.decrypt(nonce, chunk, adata)
+
+      if is_last:
+        break
+
+  return io.BufferedReader(_Unchunked(Generate()))
+
+
+# We use 12 bytes (96 bits) as it is the recommended IV length by NIST for best
+# performance [1]. See AESGCM documentation for more details.
+#
+# [1]: https://csrc.nist.gov/publications/detail/sp/800-38d/final
+_AEAD_NONCE_SIZE = 12
+
+# Because chunk size is crucial to the security of the whole procedure, we don't
+# let users pick their own chunk size. Instead, we use a fixed-size chunks of
+# 4 mebibytes.
+_AEAD_CHUNK_SIZE = 4 * 1024 * 1024
+
+# As associated data for each encrypted chunk we use an integer denoting chunk
+# id followed by a byte with information whether this is the last chunk.
+_AEAD_ADATA_FORMAT = struct.Struct("!Q?")
+
+
 def RegisterProtoDescriptors(
     db: symbol_database.SymbolDatabase,
     *additional_descriptors: descriptor.FileDescriptor,

api_client/python/grr_api_client/utils_test.py

@@ -1,8 +1,11 @@
 #!/usr/bin/env python
 import io
+import os
 import struct
 
 from absl.testing import absltest
+from cryptography import exceptions
+from cryptography.hazmat.primitives.ciphers import aead
 
 from google.protobuf import empty_pb2
 from google.protobuf import timestamp_pb2
@@ -96,5 +99,69 @@ def testDecodeSeveralChunks(self):
     self.assertEqual(b"".join(decoded), content)
 
 
+class AEADDecryptTest(absltest.TestCase):
+
+  def testReadExact(self):
+    key = os.urandom(32)
+
+    aesgcm = aead.AESGCM(key)
+    nonce = os.urandom(utils._AEAD_NONCE_SIZE)
+    adata = utils._AEAD_ADATA_FORMAT.pack(0, True)
+    encrypted = io.BytesIO(
+        nonce + aesgcm.encrypt(nonce, b"foobarbazquxnorf", adata)
+    )
+
+    decrypted = utils.AEADDecrypt(encrypted, key)
+    self.assertEqual(decrypted.read(3), b"foo")
+    self.assertEqual(decrypted.read(3), b"bar")
+    self.assertEqual(decrypted.read(3), b"baz")
+    self.assertEqual(decrypted.read(3), b"qux")
+    self.assertEqual(decrypted.read(4), b"norf")
+
+    self.assertEqual(decrypted.read(), b"")
+
+  def testIncorrectNonceLength(self):
+    key = os.urandom(32)
+
+    buf = io.BytesIO()
+
+    nonce = os.urandom(utils._AEAD_NONCE_SIZE - 1)
+    buf.write(nonce)
+    buf.seek(0, io.SEEK_SET)
+
+    with self.assertRaisesRegex(EOFError, "nonce length"):
+      utils.AEADDecrypt(buf, key).read()
+
+  def testIncorrectTag(self):
+    key = os.urandom(32)
+    aesgcm = aead.AESGCM(key)
+
+    buf = io.BytesIO()
+
+    nonce = os.urandom(utils._AEAD_NONCE_SIZE)
+    buf.write(nonce)
+    buf.write(aesgcm.encrypt(nonce, b"foo", b"QUUX"))
+    buf.seek(0, io.SEEK_SET)
+
+    with self.assertRaises(exceptions.InvalidTag):
+      utils.AEADDecrypt(buf, key).read()
+
+  def testIncorrectData(self):
+    key = os.urandom(32)
+    aesgcm = aead.AESGCM(key)
+
+    buf = io.BytesIO()
+
+    nonce = os.urandom(utils._AEAD_NONCE_SIZE)
+    adata = utils._AEAD_ADATA_FORMAT.pack(0, True)
+    buf.write(nonce)
+    buf.write(aesgcm.encrypt(nonce, b"foo", adata))
+    buf.getbuffer()[-1] ^= 0b10101010  # Corrupt last byte.
+    buf.seek(0, io.SEEK_SET)
+
+    with self.assertRaises(exceptions.InvalidTag):
+      utils.AEADDecrypt(buf, key).read()
+
+
 if __name__ == "__main__":
   absltest.main()