flatbuffers - use eager list readers where necessary

Author: vaind

generator/lib/src/code_chunks.dart

@@ -132,20 +132,24 @@ class CodeChunks {
       String fbReader;
       switch (p.type) {
         case OBXPropertyType.ByteVector:
-          fbReader = 'fb.ListReader<int>(fb.Int8Reader())';
           if (['Int8List', 'Uint8List'].contains(p.dartFieldType)) {
+            // No need for the eager reader here. We need to call fromList()
+            // constructor anyway - there's no Int8List.generate() factory.
+            fbReader = 'fb.ListReader<int>(fb.Int8Reader())';
             return '''{
              final list = ${fbReader}.vTableGet(buffer, rootOffset, ${propertyFlatBuffersvTableOffset(p)});
              object.${propertyFieldName(p)} = list == null ? null : ${p.dartFieldType}.fromList(list);
            }''';
+          } else {
+            fbReader = 'EagerListReader<int>(fb.Int8Reader())';
           }
           break;
         case OBXPropertyType.Relation:
           fbReader = 'fb.${_propertyFlatBuffersType[p.type]}Reader()';
           return 'object.${propertyFieldName(p)}.targetId = ${fbReader}.vTableGet(buffer, rootOffset, ${propertyFlatBuffersvTableOffset(p)});'
               '\n object.${propertyFieldName(p)}.attach(store);';
         case OBXPropertyType.StringVector:
-          fbReader = 'fb.ListReader<String>(fb.StringReader())';
+          fbReader = 'EagerListReader<String>(fb.StringReader())';
           break;
         default:
           fbReader = 'fb.${_propertyFlatBuffersType[p.type]}Reader()';

objectbox/lib/flatbuffers/flat_buffers.dart

@@ -39,6 +39,8 @@ typedef void StructBuilder();
 class BufferContext {
   final ByteData _buffer;
 
+  ByteData get buffer => _buffer;
+
   factory BufferContext.fromBytes(List<int> byteList) {
     Uint8List uint8List = _asUint8List(byteList);
     ByteData buf = new ByteData.view(uint8List.buffer, uint8List.offsetInBytes);

objectbox/lib/internal.dart

@@ -2,6 +2,7 @@
 /// Don't import into your own code, use 'objectbox.dart' instead.
 library objectbox_internal;
 
+export 'src/fb_readers.dart';
 export 'src/model.dart';
 export 'src/modelinfo/index.dart';
 export 'src/query/query.dart'

objectbox/lib/src/fb_readers.dart

@@ -0,0 +1,26 @@
+import 'dart:typed_data';
+
+import '../flatbuffers/flat_buffers.dart';
+
+/// Implements eager FlatBuffers list reader (default [ListReader] is lazy).
+class EagerListReader<E> extends Reader<List<E>> {
+  final Reader<E> _elementReader;
+
+  /// Create a reader with the given element reader
+  const EagerListReader(this._elementReader);
+
+  /// offset size (uint32), see [ListReader]
+  @override
+  int get size => 4;
+
+  @override
+  List<E> read(BufferContext bc, int offset) {
+    final listOffset = bc.derefObject(offset);
+    final length = bc.buffer.getUint32(listOffset, Endian.little);
+    return List<E>.generate(
+        length,
+        (int index) => _elementReader.read(
+            bc, listOffset + size + _elementReader.size * index),
+        growable: true);
+  }
+}

objectbox/test/entity.dart

@@ -79,6 +79,24 @@ class TestEntity {
       this.tUint8List,
       this.ignore});
 
+  TestEntity.filled({
+    this.id,
+    this.tString = 'Foo',
+    this.tBool = true,
+    this.tByte = 42,
+    this.tChar = 24,
+    this.tShort = 1234,
+    this.tInt = 123456789,
+    this.tLong = 123456789123456789,
+    this.tFloat = 4.5,
+    this.tDouble = 2.3,
+  }) {
+    tStrings = ['foo', 'bar'];
+    tByteList = [1, 2, 3];
+    tInt8List = Int8List.fromList([-4, 5, 6]);
+    tUint8List = Uint8List.fromList([7, 8, 9]);
+  }
+
   TestEntity.ignoredExcept(this.tInt) {
     omit = -1;
     disregard = 1;

objectbox/test/flatbuffers_test.dart

@@ -1,9 +1,14 @@
 import 'dart:ffi';
 import 'dart:typed_data';
 
-import 'package:test/test.dart';
 import 'package:flat_buffers/flat_buffers.dart' as fb_upstream;
+import 'package:objectbox/internal.dart';
 import 'package:objectbox/src/bindings/flatbuffers.dart';
+import 'package:test/test.dart';
+
+import 'entity.dart';
+import 'objectbox.g.dart';
+import 'test_env.dart';
 
 Uint8List addFbData(dynamic fbb) {
   fbb.startTable();
@@ -38,4 +43,87 @@ void main() {
       fb1.clear();
     });
   });
+
+  final bytesSum =
+      (ByteData data) => data.buffer.asInt8List().reduce((v, e) => v + e);
+
+  test('allocator', () {
+    final allocator = Allocator();
+
+    final buf1 = allocator.allocate(1024);
+    allocator.clear(buf1, true);
+
+    final buf2 = allocator.allocate(1024);
+    allocator.clear(buf2, true);
+
+    expect(bytesSum(buf1), isZero);
+    expect(bytesSum(buf2), isZero);
+
+    buf2.setInt8(42, 1);
+    expect(bytesSum(buf1), isZero);
+    expect(bytesSum(buf2), 1);
+
+    allocator.clear(buf2, true);
+    expect(bytesSum(buf2), isZero);
+
+    allocator.deallocate(buf1);
+    allocator.freeAll();
+  });
+
+  // Note: only checks content initialized by TestEntity.filled
+  void checkSameEntities(TestEntity a, TestEntity b) {
+    expect(a.tString, b.tString);
+    expect(a.tBool, b.tBool);
+    expect(a.tByte, b.tByte);
+    expect(a.tChar, b.tChar);
+    expect(a.tShort, b.tShort);
+    expect(a.tInt, b.tInt);
+    expect(a.tLong, b.tLong);
+    expect(a.tFloat, b.tFloat);
+    expect(a.tDouble, b.tDouble);
+    expect(a.tStrings, b.tStrings);
+    expect(a.tByteList, b.tByteList);
+    expect(a.tInt8List, b.tInt8List);
+    expect(a.tUint8List, b.tUint8List);
+  }
+
+  test('generated code', () {
+    final env = TestEnv('fb');
+
+    final binding = getObjectBoxModel().bindings[TestEntity]
+        as EntityDefinition<TestEntity>;
+
+    final source = TestEntity.filled();
+
+    final fb1 = BuilderWithCBuffer();
+    binding.objectToFB(source, fb1.fbb);
+    final fbData = fb1.bufPtr.cast<Uint8>().asTypedList(fb1.fbb.size);
+
+    // must have the same content after reading back
+    final target = binding.objectFromFB(env.store, fbData);
+
+    // Note: we don't do check yet, because the default flatbuffers reader
+    // reads lists lazily, on the first access and this would cause the next
+    // [checkSameEntities()] after clearing the buffer to also pass.
+    // checkSameEntities(target, source);
+
+    // explicitly clear the allocated memory
+    fbMemset(fb1.bufPtr.cast<Uint8>(), 0, fbData.lengthInBytes);
+    // fbData is now cleared as well, it's not a copy
+    expect(bytesSum(fbData.buffer.asByteData()), isZero);
+
+    // clearing the data must not affect already read objects
+    // Note: it previously did because of upstream flatbuffers lazy reading
+    checkSameEntities(target, source);
+
+    // must be empty after reading again
+    checkSameEntities(binding.objectFromFB(env.store, fbData), TestEntity());
+
+    // note: accessing fbData after fb1.clear() is illegal (memory is freed)
+    fb1.clear();
+    env.close();
+
+    // clearing the data must not affect already read objects
+    checkSameEntities(target, source);
+  });
 }