fix: use Uint8Array instead of Buffer (#1146)

Author: Uzlopak

src/concat-uint8array.ts

@@ -0,0 +1,24 @@
+export function concatUint8Array(data: Uint8Array[]): Uint8Array {
+  if (data.length === 0) {
+    // no data received
+    return new Uint8Array(0);
+  }
+
+  let totalLength = 0;
+  for (let i = 0; i < data.length; i++) {
+    totalLength += data[i].length;
+  }
+  if (totalLength === 0) {
+    // no data received
+    return new Uint8Array(0);
+  }
+
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (let i = 0; i < data.length; i++) {
+    result.set(data[i], offset);
+    offset += data[i].length;
+  }
+
+  return result;
+}

src/middleware/node/get-payload.ts

@@ -1,43 +1,43 @@
-// remove type imports from http for Deno compatibility
-// see https://github.com/octokit/octokit.js/issues/2075#issuecomment-817361886
-// import type { IncomingMessage } from "node:http";
-// declare module "node:http" {
-//    interface IncomingMessage {
-//      body?: string;
-//    }
-// }
+import { concatUint8Array } from "../../concat-uint8array.ts";
+
 type IncomingMessage = any;
 
-export function getPayload(request: IncomingMessage): Promise<string> {
+const textDecoder = new TextDecoder("utf-8", { fatal: false });
+const decode = textDecoder.decode.bind(textDecoder);
+
+export async function getPayload(request: IncomingMessage): Promise<string> {
   if (
     typeof request.body === "object" &&
     "rawBody" in request &&
-    request.rawBody instanceof Buffer
+    request.rawBody instanceof Uint8Array
   ) {
-    // The body is already an Object and rawBody is a Buffer (e.g. GCF)
-    return Promise.resolve(request.rawBody.toString("utf8"));
+    // The body is already an Object and rawBody is a Buffer/Uint8Array (e.g. GCF)
+    return decode(request.rawBody);
   } else if (typeof request.body === "string") {
     // The body is a String (e.g. Lambda)
-    return Promise.resolve(request.body);
+    return request.body;
   }
 
+  // We need to load the payload from the request (normal case of Node.js server)
+  const payload = await getPayloadFromRequestStream(request);
+  return decode(payload);
+}
+
+export function getPayloadFromRequestStream(
+  request: IncomingMessage,
+): Promise<Uint8Array> {
   // We need to load the payload from the request (normal case of Node.js server)
   return new Promise((resolve, reject) => {
-    let data: Buffer[] = [];
+    let data: Uint8Array[] = [];
 
     request.on("error", (error: Error) =>
       reject(new AggregateError([error], error.message)),
     );
-    request.on("data", (chunk: Buffer) => data.push(chunk));
-    request.on("end", () =>
-      // setImmediate improves the throughput by reducing the pressure from
-      // the event loop
-      setImmediate(
-        resolve,
-        data.length === 1
-          ? data[0].toString("utf8")
-          : Buffer.concat(data).toString("utf8"),
-      ),
-    );
+    request.on("data", data.push.bind(data));
+    request.on("end", () => {
+      const result = concatUint8Array(data);
+      // Switch to queue microtask when we want to support bun and deno
+      setImmediate(resolve, result);
+    });
   });
 }

test/integration/get-payload.test.ts

@@ -70,7 +70,7 @@ describe("getPayload", () => {
     const promise = getPayload(request);
 
     // we emit data, to ensure that the body attribute is preferred
-    request.emit("data", "bar");
+    request.emit("data", Buffer.from("bar"));
     request.emit("end");
 
     expect(await promise).toEqual("foo");
@@ -85,9 +85,20 @@ describe("getPayload", () => {
     const promise = getPayload(request);
 
     // we emit data, to ensure that the body attribute is preferred
-    request.emit("data", "bar");
+    request.emit("data", Buffer.from("bar"));
     request.emit("end");
 
     expect(await promise).toEqual("bar");
   });
+
+  it("should not throw an error if non-valid utf-8 payload was received", async () => {
+    const request = new EventEmitter();
+
+    const promise = getPayload(request);
+
+    request.emit("data", new Uint8Array([226]));
+    request.emit("end");
+
+    expect(await promise).toEqual("�");
+  });
 });

test/unit/concat-uint8array.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, assert } from "vitest";
+
+import { concatUint8Array } from "../../src/concat-uint8array.ts";
+
+describe("concatUint8Array", () => {
+  it("returns an empty array when no data is provided", () => {
+    const result = concatUint8Array([]);
+    assert(result.length === 0);
+  });
+
+  it("returns a single Uint8Array when one Uint8Array is provided", () => {
+    const data = new Uint8Array([1, 2, 3]);
+    const result = concatUint8Array([data]);
+    assert(data !== result);
+    assert(result.length === 3);
+    assert(result[0] === 1);
+    assert(result[1] === 2);
+    assert(result[2] === 3);
+  });
+
+  it("concatenates multiple Uint8Arrays into a single Uint8Array", () => {
+    const data = [
+      new Uint8Array([1, 2]),
+      new Uint8Array([3, 4]),
+      new Uint8Array([5, 6]),
+    ];
+    const result = concatUint8Array(data);
+
+    assert(data[0] !== result);
+    assert(data[1] !== result);
+    assert(data[2] !== result);
+
+    assert(result.length === 6);
+    assert(result[0] === 1);
+    assert(result[1] === 2);
+    assert(result[2] === 3);
+    assert(result[3] === 4);
+    assert(result[4] === 5);
+    assert(result[5] === 6);
+  });
+});