fix: Allow to open database files >2GiB and optimize memory consumption for reading files (#878)

* Update index.ts

* Update fs.ts

* New contributor

* Fix ts

* allow to publish again while master branch is updated

* Update package.json

* Update index.ts

* Update tests

Author: MarianoFacundoArch

package.json

@@ -23,7 +23,8 @@
     "Thomas Birke @quafzi <[email protected]>",
     "Afzaal Ameer @afzaalace",
     "Andrew N Golovkov @AndorCS",
-    "Gregory Oschwald @oschwald"
+    "Gregory Oschwald @oschwald",
+    "Mariano Facundo Scigliano @MarianoFacundoArch"
   ],
   "dependencies": {
     "mmdb-lib": "2.1.1",

src/fs.ts

@@ -5,4 +5,5 @@ export default {
   existsSync: fs.existsSync,
   readFile: util.promisify(fs.readFile),
   watchFile: fs.watchFile,
+  createReadStream: fs.createReadStream, 
 };

src/index.test.ts

@@ -18,7 +18,7 @@ describe('index', () => {
     sandbox.stub(fs, 'watchFile').callsFake((paramA, paramB, cb) => {
       watchHandler = cb;
     });
-    sandbox.spy(fs, 'readFile');
+    sandbox.spy(fs, 'createReadStream');
   });
   afterEach(() => {
     sandbox.restore();
@@ -51,17 +51,17 @@ describe('index', () => {
       const lookup = await maxmind.open(dbPath, options);
       assert(lookup.get('2001:230::'));
       assert((fs.watchFile as SinonSpy).calledOnce);
-      assert((fs.readFile as SinonSpy).calledOnce);
+      assert((fs.createReadStream as SinonSpy).calledOnce);
     });
 
     it('should work with auto updates', async () => {
       const options = { watchForUpdates: true };
       const lookup = await maxmind.open(dbPath, options);
       assert(lookup.get('2001:230::'));
       assert((fs.watchFile as SinonSpy).calledOnce);
-      assert((fs.readFile as SinonSpy).calledOnce);
+      assert((fs.createReadStream as SinonSpy).calledOnce);
       await watchHandler();
-      assert((fs.readFile as SinonSpy).calledTwice);
+      assert((fs.createReadStream as SinonSpy).calledTwice);
     });
 
     it('should work with auto updates and call specified hook', async () => {

src/index.ts

@@ -17,14 +17,36 @@ export interface OpenOpts {
   watchForUpdatesHook?: Callback;
 }
 
+const readFile = async (filepath: string): Promise<Buffer> => {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    const stream = fs.createReadStream(filepath, {
+      highWaterMark: 64 * 1024 * 1024, // 64 MB chunks
+    });
+
+    stream.on('data', (chunk: Buffer) => {
+      chunks.push(chunk);
+    });
+
+    stream.on('end', () => {
+      resolve(Buffer.concat(chunks));
+    });
+
+    stream.on('error', (err) => {
+      reject(err);
+    });
+  });
+};
+
+
 export const open = async <T extends Response>(
   filepath: string,
   opts?: OpenOpts,
   cb?: Callback
 ): Promise<Reader<T>> => {
   assert(!cb, utils.legacyErrorMessage);
 
-  const database = await fs.readFile(filepath);
+  const database = await readFile(filepath);
 
   if (isGzip(database)) {
     throw new Error(
@@ -63,7 +85,7 @@ export const open = async <T extends Response>(
       if (!(await waitExists())) {
         return;
       }
-      const updatedDatabase = await fs.readFile(filepath);
+      const updatedDatabase = await readFile(filepath);
       cache.clear();
       reader.load(updatedDatabase);
       if (opts.watchForUpdatesHook) {