diff --git a/src/fs.ts b/src/fs.ts index 004fdc2e..0d256a4f 100644 --- a/src/fs.ts +++ b/src/fs.ts @@ -5,5 +5,6 @@ export default { existsSync: fs.existsSync, readFile: util.promisify(fs.readFile), watchFile: fs.watchFile, - createReadStream: fs.createReadStream, + createReadStream: fs.createReadStream, + stat: util.promisify(fs.stat), }; diff --git a/src/index.test.ts b/src/index.test.ts index e41df007..d871b004 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -19,6 +19,7 @@ describe('index', () => { watchHandler = cb; }); sandbox.spy(fs, 'createReadStream'); + sandbox.spy(fs, 'readFile'); }); afterEach(() => { sandbox.restore(); @@ -51,7 +52,7 @@ describe('index', () => { const lookup = await maxmind.open(dbPath, options); assert(lookup.get('2001:230::')); assert((fs.watchFile as SinonSpy).calledOnce); - assert((fs.createReadStream as SinonSpy).calledOnce); + assert((fs.readFile as SinonSpy).calledOnce); }); it('should work with auto updates', async () => { @@ -59,9 +60,9 @@ describe('index', () => { const lookup = await maxmind.open(dbPath, options); assert(lookup.get('2001:230::')); assert((fs.watchFile as SinonSpy).calledOnce); - assert((fs.createReadStream as SinonSpy).calledOnce); + assert((fs.readFile as SinonSpy).calledOnce); await watchHandler(); - assert((fs.createReadStream as SinonSpy).calledTwice); + assert((fs.readFile as SinonSpy).calledTwice); }); it('should work with auto updates and call specified hook', async () => { diff --git a/src/index.ts b/src/index.ts index 4e5577e9..cbccf9e2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,9 @@ import ip from './ip'; import isGzip from './is-gzip'; import utils from './utils'; +const LARGE_FILE_THRESHOLD = 512 * 1024 * 1024; +const STREAM_WATERMARK = 8 * 1024 * 1024; + type Callback = () => void; export interface OpenOpts { @@ -17,27 +20,48 @@ export interface OpenOpts { watchForUpdatesHook?: Callback; } -const readFile = async (filepath: string): Promise => { - return new Promise((resolve, reject) => { - const chunks: Buffer[] = []; +/** + * Read large file in chunks. + * + * Reason it's not used for all file sizes is that it's slower than fs.readFile and uses + * a bit more memory due to the buffer operations. + * + * Node seems to have a limit of 2GB for fs.readFileSync, so we need to use streams for + * larger files. + * + * @param filepath + * @param size + * @returns + */ +const readLargeFile = async (filepath: string, size: number): Promise => + new Promise((resolve, reject) => { + let buffer = Buffer.allocUnsafe(size); + let offset = 0; const stream = fs.createReadStream(filepath, { - highWaterMark: 64 * 1024 * 1024, // 64 MB chunks + highWaterMark: STREAM_WATERMARK, }); stream.on('data', (chunk: Buffer) => { - chunks.push(chunk); + chunk.copy(buffer, offset); + offset += chunk.length; }); stream.on('end', () => { - resolve(Buffer.concat(chunks)); + stream.close(); + resolve(buffer); }); stream.on('error', (err) => { reject(err); }); }); -}; +const readFile = async (filepath: string): Promise => { + const fstat = await fs.stat(filepath); + return fstat.size < LARGE_FILE_THRESHOLD + ? fs.readFile(filepath) + : readLargeFile(filepath, fstat.size); +}; export const open = async ( filepath: string,