fix: Fix read file fn which uses double the memory

[runk]

Solves #885

Debugging script for reference

const fs = require('fs');
const crypto = require('crypto');

const showMemoryUsage = () => {
  const memoryUsage = process.memoryUsage();

  console.log(`X: ${Math.round(memoryUsage.heapUsed / 1024 / 1024)} MB`);
  console.log('Memory Usage:');
  console.log(
    `- RSS (Resident Set Size): ${Math.round(memoryUsage.rss / 1024 / 1024)} MB`
  );
  console.log(
    `- Heap Total: ${Math.round(memoryUsage.heapTotal / 1024 / 1024)} MB`
  );
  console.log(
    `- Heap Used: ${Math.round(memoryUsage.heapUsed / 1024 / 1024)} MB`
  );
  console.log(
    `- External: ${Math.round(memoryUsage.external / 1024 / 1024)} MB`
  );
  console.log(
    `- Array Buffers: ${Math.round(memoryUsage.arrayBuffers / 1024 / 1024)} MB`
  );
};

// const stream = fs.createWriteStream('./data');
// for (let i = 0; i < 1000; i++) {
//   stream.write(crypto.randomBytes(1024 * 1024));
// }
// stream.end();

const readFile2 = (filepath, cb) => {
  return new Promise((resolve, reject) => {
    const stat = fs.statSync(filepath);
    let buffer = Buffer.allocUnsafe(stat.size);
    let offset = 0;
    const stream = fs.createReadStream(filepath, {
      highWaterMark: 8 * 1024 * 1024,
    });

    stream.on('data', (chunk) => {
      chunk.copy(buffer, offset);
      offset += chunk.length;
    });

    stream.on('end', () => {
      stream.close();
      resolve(buffer);
    });

    stream.on('error', (err) => {
      reject(err);
    });
  });
};

const readFileOriginal = async (filepath) => {
  return fs.readFileSync(filepath);
};

const main = async () => {
  // showMemoryUsage();
  const now = performance.now();
  // const buffer = await readFileOriginal('./data');

  const buffer = await readFile2('./data');

  console.log('took', performance.now() - now);
  console.log(buffer.length);

  setTimeout(() => {
    showMemoryUsage();
  }, 2000);
};

main().catch(console.error);

[github-actions]

🎉 This PR is included in version 4.3.24 🎉

The release is available on:

• npm package (@latest dist-tag)
• GitHub release

Your semantic-release bot 📦🚀

[runk]

Old implementation used a list of buffers which was driving excessive memory usage + Buffer.concat which apparently is not memory friendly as well.

Now I'm allocating resulting buffer of the size of the file, and filling it with chunks as we go.