From 2cd531b9f48fabcb49169ee55cd1a1edbaa1e4fd Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:29:17 +0000 Subject: [PATCH 1/3] feat(middleware-flexible-checksums): use Node.js native CRC32 checksum API --- .../package.json | 4 +++ ...tCrc32ChecksumAlgorithmFunction.browser.ts | 3 ++ .../src/getCrc32ChecksumAlgorithmFunction.ts | 28 +++++++++++++++++++ .../src/selectChecksumAlgorithmFunction.ts | 4 +-- yarn.lock | 2 +- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.browser.ts create mode 100644 packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts diff --git a/packages/middleware-flexible-checksums/package.json b/packages/middleware-flexible-checksums/package.json index b929c541b01b..c26fe92c2665 100644 --- a/packages/middleware-flexible-checksums/package.json +++ b/packages/middleware-flexible-checksums/package.json @@ -17,10 +17,13 @@ "main": "./dist-cjs/index.js", "module": "./dist-es/index.js", "browser": { + "./dist-es/getCrc32ChecksumAlgorithmFunction": "./dist-es/getCrc32ChecksumAlgorithmFunction.browser", "./dist-es/streams/create-read-stream-on-buffer": "./dist-es/streams/create-read-stream-on-buffer.browser" }, "react-native": { + "./dist-es/getCrc32ChecksumAlgorithmFunction": "./dist-es/getCrc32ChecksumAlgorithmFunction.browser", "./dist-es/streams/create-read-stream-on-buffer": "./dist-es/streams/create-read-stream-on-buffer.browser", + "./dist-cjs/getCrc32ChecksumAlgorithmFunction": "./dist-cjs/getCrc32ChecksumAlgorithmFunction.browser", "./dist-cjs/streams/create-read-stream-on-buffer": "./dist-cjs/streams/create-read-stream-on-buffer.browser" }, "types": "./dist-types/index.d.ts", @@ -32,6 +35,7 @@ "dependencies": { "@aws-crypto/crc32": "5.2.0", "@aws-crypto/crc32c": "5.2.0", + "@aws-crypto/util": "5.2.0", "@aws-sdk/core": "*", "@aws-sdk/types": "*", "@smithy/is-array-buffer": "^3.0.0", diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.browser.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.browser.ts new file mode 100644 index 000000000000..1c482b41706c --- /dev/null +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.browser.ts @@ -0,0 +1,3 @@ +import { AwsCrc32 } from "@aws-crypto/crc32"; + +export const getCrc32ChecksumAlgorithmFunction = () => AwsCrc32; diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts new file mode 100644 index 000000000000..8d38987a975f --- /dev/null +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts @@ -0,0 +1,28 @@ +import { AwsCrc32 } from "@aws-crypto/crc32"; +import { numToUint8 } from "@aws-crypto/util"; +import { Checksum } from "@smithy/types"; +import zlib from "zlib"; + +export const getCrc32ChecksumAlgorithmFunction = () => { + // @ts-expect-error crc32 is defined only for Node.js >v20.15.0+ and >v22.2.0+. + if (typeof zlib.crc32 === "undefined") { + return AwsCrc32; + } + + return class NodeCrc32 implements Checksum { + private checksum = 0; + + update(data: Uint8Array) { + // @ts-expect-error crc32 is defined only for Node.js >v20.15.0+ and >v22.2.0+. + this.checksum = zlib.crc32(data, this.checksum); + } + + async digest() { + return numToUint8(this.checksum); + } + + reset() { + this.checksum = 0; + } + }; +}; diff --git a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts index 39e0f192245a..74a09add4a30 100644 --- a/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts +++ b/packages/middleware-flexible-checksums/src/selectChecksumAlgorithmFunction.ts @@ -1,9 +1,9 @@ -import { AwsCrc32 } from "@aws-crypto/crc32"; import { AwsCrc32c } from "@aws-crypto/crc32c"; import { ChecksumConstructor, HashConstructor } from "@smithy/types"; import { PreviouslyResolved } from "./configuration"; import { ChecksumAlgorithm } from "./constants"; +import { getCrc32ChecksumAlgorithmFunction } from "./getCrc32ChecksumAlgorithmFunction"; /** * Returns the function that will compute the checksum for the given {@link ChecksumAlgorithm}. @@ -14,7 +14,7 @@ export const selectChecksumAlgorithmFunction = ( ): ChecksumConstructor | HashConstructor => ({ [ChecksumAlgorithm.MD5]: config.md5, - [ChecksumAlgorithm.CRC32]: AwsCrc32, + [ChecksumAlgorithm.CRC32]: getCrc32ChecksumAlgorithmFunction(), [ChecksumAlgorithm.CRC32C]: AwsCrc32c, [ChecksumAlgorithm.SHA1]: config.sha1, [ChecksumAlgorithm.SHA256]: config.sha256, diff --git a/yarn.lock b/yarn.lock index cecbf306d72d..5c0b0a027605 100644 --- a/yarn.lock +++ b/yarn.lock @@ -74,7 +74,7 @@ dependencies: tslib "^2.6.2" -"@aws-crypto/util@^5.2.0": +"@aws-crypto/util@5.2.0", "@aws-crypto/util@^5.2.0": version "5.2.0" resolved "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz" integrity sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ== From adfe83e2705ab94b6d0db8f4fd31171e40c52277 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Thu, 7 Nov 2024 23:52:49 +0000 Subject: [PATCH 2/3] test: src/getCrc32ChecksumAlgorithmFunction.spec.ts --- .../getCrc32ChecksumAlgorithmFunction.spec.ts | 48 +++++++++++++++++++ .../src/getCrc32ChecksumAlgorithmFunction.ts | 4 +- 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts new file mode 100644 index 000000000000..121ccf0dbf8f --- /dev/null +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts @@ -0,0 +1,48 @@ +import { AwsCrc32 } from "@aws-crypto/crc32"; +import { numToUint8 } from "@aws-crypto/util"; +import { describe, expect, test as it, vi } from "vitest"; +import zlib from "zlib"; + +import { getCrc32ChecksumAlgorithmFunction } from "./getCrc32ChecksumAlgorithmFunction"; + +describe(getCrc32ChecksumAlgorithmFunction.name, () => { + it("returns AwsCrc32 if zlib.crc32 is undefined", () => { + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + zlib.crc32 = undefined; + expect(getCrc32ChecksumAlgorithmFunction()).toBe(AwsCrc32); + }); + + it("returns NodeCrc32 if zlib.crc32 is defined", async () => { + const mockData = new Uint8Array([1, 2, 3]); + const mockChecksum = 42; + + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + zlib.crc32 = vi + .fn() + .mockReturnValueOnce(mockChecksum) + .mockReturnValueOnce(2 * mockChecksum); + + const crc32Fn = getCrc32ChecksumAlgorithmFunction(); + expect(crc32Fn).not.toBe(AwsCrc32); + + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + expect(zlib.crc32).not.toHaveBeenCalled(); + const crc32 = new crc32Fn(); + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + expect(zlib.crc32).not.toHaveBeenCalled(); + expect(await crc32.digest()).toEqual(numToUint8(0)); + + crc32.update(mockData); + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + expect(zlib.crc32).toHaveBeenCalledWith(mockData, 0); + expect(await crc32.digest()).toEqual(numToUint8(mockChecksum)); + + crc32.update(mockData); + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. + expect(zlib.crc32).toHaveBeenCalledWith(mockData, mockChecksum); + expect(await crc32.digest()).toEqual(numToUint8(2 * mockChecksum)); + + crc32.reset(); + expect(await crc32.digest()).toEqual(numToUint8(0)); + }); +}); diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts index 8d38987a975f..8c434b5cb312 100644 --- a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts @@ -4,7 +4,7 @@ import { Checksum } from "@smithy/types"; import zlib from "zlib"; export const getCrc32ChecksumAlgorithmFunction = () => { - // @ts-expect-error crc32 is defined only for Node.js >v20.15.0+ and >v22.2.0+. + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. if (typeof zlib.crc32 === "undefined") { return AwsCrc32; } @@ -13,7 +13,7 @@ export const getCrc32ChecksumAlgorithmFunction = () => { private checksum = 0; update(data: Uint8Array) { - // @ts-expect-error crc32 is defined only for Node.js >v20.15.0+ and >v22.2.0+. + // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. this.checksum = zlib.crc32(data, this.checksum); } From 2cffda4ce5b73cbd9fbc56575d140e069425e3b3 Mon Sep 17 00:00:00 2001 From: "Kamat, Trivikram" <16024985+trivikr@users.noreply.github.com> Date: Fri, 8 Nov 2024 00:51:21 +0000 Subject: [PATCH 3/3] fix: build error --- .../src/getCrc32ChecksumAlgorithmFunction.spec.ts | 5 ++--- .../src/getCrc32ChecksumAlgorithmFunction.ts | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts index 121ccf0dbf8f..053895b815f0 100644 --- a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.spec.ts @@ -1,14 +1,13 @@ import { AwsCrc32 } from "@aws-crypto/crc32"; import { numToUint8 } from "@aws-crypto/util"; import { describe, expect, test as it, vi } from "vitest"; -import zlib from "zlib"; +import * as zlib from "zlib"; import { getCrc32ChecksumAlgorithmFunction } from "./getCrc32ChecksumAlgorithmFunction"; describe(getCrc32ChecksumAlgorithmFunction.name, () => { it("returns AwsCrc32 if zlib.crc32 is undefined", () => { - // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. - zlib.crc32 = undefined; + vi.mock("zlib", () => ({ crc32: undefined })); expect(getCrc32ChecksumAlgorithmFunction()).toBe(AwsCrc32); }); diff --git a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts index 8c434b5cb312..3803ccfdedae 100644 --- a/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts +++ b/packages/middleware-flexible-checksums/src/getCrc32ChecksumAlgorithmFunction.ts @@ -1,7 +1,7 @@ import { AwsCrc32 } from "@aws-crypto/crc32"; import { numToUint8 } from "@aws-crypto/util"; import { Checksum } from "@smithy/types"; -import zlib from "zlib"; +import * as zlib from "zlib"; export const getCrc32ChecksumAlgorithmFunction = () => { // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0. @@ -10,7 +10,7 @@ export const getCrc32ChecksumAlgorithmFunction = () => { } return class NodeCrc32 implements Checksum { - private checksum = 0; + checksum = 0; update(data: Uint8Array) { // @ts-expect-error crc32 is defined only for Node.js >=v20.15.0 and >=v22.2.0.