From f097ca694b2fcacf443cc502ea37043f78590d7d Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Fri, 26 Sep 2025 09:47:22 -0400 Subject: [PATCH 1/9] DuckDBRowReader: Add class to stream DuckDB results efficiently There currently exists the `DuckDBResultReader`, but the issue that we're running into is that this processes chunks into an internal buffer that's never cleared. For large result sets, this buffer builds and builds until we run out of memory. The case that we have is that we just need the result set row by row without needing to compute any of the column data. This new class does less than `DuckDBResultReader` by not storing the results of the query in the underlying buffer. Instead, the results of the chunk are passed back to the caller where the caller can handle the chunk as they wish. The interface for processing rows is similar to `DuckDBRowReader` where the caller can still use converters as they wish to process their data. --- api/src/DuckDBConnection.ts | 8 ++ api/src/DuckDBRowReader.ts | 182 ++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 api/src/DuckDBRowReader.ts diff --git a/api/src/DuckDBConnection.ts b/api/src/DuckDBConnection.ts index eddc433d..395d39d8 100644 --- a/api/src/DuckDBConnection.ts +++ b/api/src/DuckDBConnection.ts @@ -11,6 +11,7 @@ import { DuckDBResultReader } from './DuckDBResultReader'; import { DuckDBScalarFunction } from './DuckDBScalarFunction'; import { DuckDBType } from './DuckDBType'; import { DuckDBValue } from './values'; +import { DuckDBRowReader } from "./DuckDBRowReader"; export class DuckDBConnection { private readonly connection: duckdb.Connection; @@ -87,6 +88,13 @@ export class DuckDBConnection { await reader.readUntil(targetRowCount); return reader; } + public async rowReaderStream( + sql: string, + values?: DuckDBValue[] | Record, + types?: DuckDBType[] | Record + ): Promise { + return new DuckDBRowReader(await this.run(sql, values, types)); + } public async stream( sql: string, values?: DuckDBValue[] | Record, diff --git a/api/src/DuckDBRowReader.ts b/api/src/DuckDBRowReader.ts new file mode 100644 index 00000000..d2c3b489 --- /dev/null +++ b/api/src/DuckDBRowReader.ts @@ -0,0 +1,182 @@ +import { convertRowObjectsFromChunks } from './convertRowObjectsFromChunks'; +import { convertRowsFromChunks } from './convertRowsFromChunks'; +import { DuckDBDataChunk } from './DuckDBDataChunk'; +import { DuckDBLogicalType } from './DuckDBLogicalType'; +import { DuckDBResult } from './DuckDBResult'; +import { DuckDBType } from './DuckDBType'; +import { DuckDBTypeId } from './DuckDBTypeId'; +import { DuckDBValueConverter } from './DuckDBValueConverter'; +import { ResultReturnType, StatementType } from './enums'; +import { getRowObjectsFromChunks } from './getRowObjectsFromChunks'; +import { getRowsFromChunks } from './getRowsFromChunks'; +import { JS } from './JS'; +import { JSDuckDBValueConverter } from './JSDuckDBValueConverter'; +import { Json } from './Json'; +import { JsonDuckDBValueConverter } from './JsonDuckDBValueConverter'; +import { DuckDBValue } from './values'; + +/** Used to read the results row by row in a memory efficient manner */ +export class DuckDBRowReader { + private readonly result: DuckDBResult; + private chunks: AsyncGenerator; + private currentRowCount_: number; + private done_: boolean; + + constructor(result: DuckDBResult) { + this.result = result; + this.currentRowCount_ = 0; + this.done_ = false; + this.chunks = this.chunksGenerator(); + } + + private async *chunksGenerator(): AsyncGenerator< + DuckDBDataChunk, + void, + undefined + > { + while (!this.done_) { + const chunk = await this.result.fetchChunk(); + if (chunk && chunk.rowCount > 0) { + this.currentRowCount_ += chunk.rowCount; + yield chunk; + } else { + this.done_ = true; + } + } + } + + public get returnType(): ResultReturnType { + return this.result.returnType; + } + + public get statementType(): StatementType { + return this.result.statementType; + } + + public get columnCount(): number { + return this.result.columnCount; + } + + public columnName(columnIndex: number): string { + return this.result.columnName(columnIndex); + } + + public columnNames(): string[] { + return this.result.columnNames(); + } + + public deduplicatedColumnNames(): string[] { + return this.result.deduplicatedColumnNames(); + } + + public columnTypeId(columnIndex: number): DuckDBTypeId { + return this.result.columnTypeId(columnIndex); + } + + public columnLogicalType(columnIndex: number): DuckDBLogicalType { + return this.result.columnLogicalType(columnIndex); + } + + public columnType(columnIndex: number): DuckDBType { + return this.result.columnType(columnIndex); + } + + public columnTypeJson(columnIndex: number): Json { + return this.result.columnTypeJson(columnIndex); + } + + public columnTypes(): DuckDBType[] { + return this.result.columnTypes(); + } + + public columnTypesJson(): Json { + return this.result.columnTypesJson(); + } + + public columnNamesAndTypesJson(): Json { + return this.result.columnNamesAndTypesJson(); + } + + public columnNameAndTypeObjectsJson(): Json { + return this.result.columnNameAndTypeObjectsJson(); + } + + public get rowsChanged(): number { + return this.result.rowsChanged; + } + + /** Total number of rows read so far. Call `readAll` or `readUntil` to read rows. */ + public get currentRowCount() { + return this.currentRowCount_; + } + + /** Whether reading is done, that is, there are no more rows to read. */ + public get done() { + return this.done_; + } + + /** + * Internal iterator used to return the DuckDBValue rows. The public functions on + * this class are wrappers around this generator so that the caller can specify + * how they want to consume the rows. + */ + private async *getRows(): AsyncGenerator { + for await (const chunk of this.chunks) { + yield getRowsFromChunks([chunk]); + } + } + + public async *convertRows( + converter: DuckDBValueConverter + ): AsyncGenerator<(T | null)[][], void, undefined> { + for await (const chunk of this.chunks) { + yield convertRowsFromChunks([chunk], converter); + } + } + + public getRowsJS(): AsyncGenerator { + return this.convertRows(JSDuckDBValueConverter); + } + + public getRowsJson(): AsyncGenerator { + return this.convertRows(JsonDuckDBValueConverter); + } + + public async *getRowObjects(): AsyncGenerator< + Record[], + void, + undefined + > { + for await (const chunk of this.chunks) { + yield getRowObjectsFromChunks([chunk], this.deduplicatedColumnNames()); + } + } + + public async *convertNextRowObjects( + converter: DuckDBValueConverter + ): AsyncGenerator[], void, undefined> { + for await (const chunk of this.chunks) { + yield convertRowObjectsFromChunks( + [chunk], + this.deduplicatedColumnNames(), + converter, + ); + } + } + + public getRowObjectsJS(): AsyncGenerator< + Record[], + void, + undefined + > { + return this.convertNextRowObjects(JSDuckDBValueConverter); + } + + public getRowObjectsJson(): AsyncGenerator< + Record[], + void, + undefined + > { + return this.convertNextRowObjects(JsonDuckDBValueConverter); + } +} From f6b56b3dcad0a672184200506f48e50f903439d7 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sat, 27 Sep 2025 08:01:45 -0400 Subject: [PATCH 2/9] DuckDBRowReader: Remove class and usage in DuckDBConnection --- api/src/DuckDBConnection.ts | 8 -- api/src/DuckDBRowReader.ts | 182 ------------------------------------ 2 files changed, 190 deletions(-) delete mode 100644 api/src/DuckDBRowReader.ts diff --git a/api/src/DuckDBConnection.ts b/api/src/DuckDBConnection.ts index 395d39d8..eddc433d 100644 --- a/api/src/DuckDBConnection.ts +++ b/api/src/DuckDBConnection.ts @@ -11,7 +11,6 @@ import { DuckDBResultReader } from './DuckDBResultReader'; import { DuckDBScalarFunction } from './DuckDBScalarFunction'; import { DuckDBType } from './DuckDBType'; import { DuckDBValue } from './values'; -import { DuckDBRowReader } from "./DuckDBRowReader"; export class DuckDBConnection { private readonly connection: duckdb.Connection; @@ -88,13 +87,6 @@ export class DuckDBConnection { await reader.readUntil(targetRowCount); return reader; } - public async rowReaderStream( - sql: string, - values?: DuckDBValue[] | Record, - types?: DuckDBType[] | Record - ): Promise { - return new DuckDBRowReader(await this.run(sql, values, types)); - } public async stream( sql: string, values?: DuckDBValue[] | Record, diff --git a/api/src/DuckDBRowReader.ts b/api/src/DuckDBRowReader.ts deleted file mode 100644 index d2c3b489..00000000 --- a/api/src/DuckDBRowReader.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { convertRowObjectsFromChunks } from './convertRowObjectsFromChunks'; -import { convertRowsFromChunks } from './convertRowsFromChunks'; -import { DuckDBDataChunk } from './DuckDBDataChunk'; -import { DuckDBLogicalType } from './DuckDBLogicalType'; -import { DuckDBResult } from './DuckDBResult'; -import { DuckDBType } from './DuckDBType'; -import { DuckDBTypeId } from './DuckDBTypeId'; -import { DuckDBValueConverter } from './DuckDBValueConverter'; -import { ResultReturnType, StatementType } from './enums'; -import { getRowObjectsFromChunks } from './getRowObjectsFromChunks'; -import { getRowsFromChunks } from './getRowsFromChunks'; -import { JS } from './JS'; -import { JSDuckDBValueConverter } from './JSDuckDBValueConverter'; -import { Json } from './Json'; -import { JsonDuckDBValueConverter } from './JsonDuckDBValueConverter'; -import { DuckDBValue } from './values'; - -/** Used to read the results row by row in a memory efficient manner */ -export class DuckDBRowReader { - private readonly result: DuckDBResult; - private chunks: AsyncGenerator; - private currentRowCount_: number; - private done_: boolean; - - constructor(result: DuckDBResult) { - this.result = result; - this.currentRowCount_ = 0; - this.done_ = false; - this.chunks = this.chunksGenerator(); - } - - private async *chunksGenerator(): AsyncGenerator< - DuckDBDataChunk, - void, - undefined - > { - while (!this.done_) { - const chunk = await this.result.fetchChunk(); - if (chunk && chunk.rowCount > 0) { - this.currentRowCount_ += chunk.rowCount; - yield chunk; - } else { - this.done_ = true; - } - } - } - - public get returnType(): ResultReturnType { - return this.result.returnType; - } - - public get statementType(): StatementType { - return this.result.statementType; - } - - public get columnCount(): number { - return this.result.columnCount; - } - - public columnName(columnIndex: number): string { - return this.result.columnName(columnIndex); - } - - public columnNames(): string[] { - return this.result.columnNames(); - } - - public deduplicatedColumnNames(): string[] { - return this.result.deduplicatedColumnNames(); - } - - public columnTypeId(columnIndex: number): DuckDBTypeId { - return this.result.columnTypeId(columnIndex); - } - - public columnLogicalType(columnIndex: number): DuckDBLogicalType { - return this.result.columnLogicalType(columnIndex); - } - - public columnType(columnIndex: number): DuckDBType { - return this.result.columnType(columnIndex); - } - - public columnTypeJson(columnIndex: number): Json { - return this.result.columnTypeJson(columnIndex); - } - - public columnTypes(): DuckDBType[] { - return this.result.columnTypes(); - } - - public columnTypesJson(): Json { - return this.result.columnTypesJson(); - } - - public columnNamesAndTypesJson(): Json { - return this.result.columnNamesAndTypesJson(); - } - - public columnNameAndTypeObjectsJson(): Json { - return this.result.columnNameAndTypeObjectsJson(); - } - - public get rowsChanged(): number { - return this.result.rowsChanged; - } - - /** Total number of rows read so far. Call `readAll` or `readUntil` to read rows. */ - public get currentRowCount() { - return this.currentRowCount_; - } - - /** Whether reading is done, that is, there are no more rows to read. */ - public get done() { - return this.done_; - } - - /** - * Internal iterator used to return the DuckDBValue rows. The public functions on - * this class are wrappers around this generator so that the caller can specify - * how they want to consume the rows. - */ - private async *getRows(): AsyncGenerator { - for await (const chunk of this.chunks) { - yield getRowsFromChunks([chunk]); - } - } - - public async *convertRows( - converter: DuckDBValueConverter - ): AsyncGenerator<(T | null)[][], void, undefined> { - for await (const chunk of this.chunks) { - yield convertRowsFromChunks([chunk], converter); - } - } - - public getRowsJS(): AsyncGenerator { - return this.convertRows(JSDuckDBValueConverter); - } - - public getRowsJson(): AsyncGenerator { - return this.convertRows(JsonDuckDBValueConverter); - } - - public async *getRowObjects(): AsyncGenerator< - Record[], - void, - undefined - > { - for await (const chunk of this.chunks) { - yield getRowObjectsFromChunks([chunk], this.deduplicatedColumnNames()); - } - } - - public async *convertNextRowObjects( - converter: DuckDBValueConverter - ): AsyncGenerator[], void, undefined> { - for await (const chunk of this.chunks) { - yield convertRowObjectsFromChunks( - [chunk], - this.deduplicatedColumnNames(), - converter, - ); - } - } - - public getRowObjectsJS(): AsyncGenerator< - Record[], - void, - undefined - > { - return this.convertNextRowObjects(JSDuckDBValueConverter); - } - - public getRowObjectsJson(): AsyncGenerator< - Record[], - void, - undefined - > { - return this.convertNextRowObjects(JsonDuckDBValueConverter); - } -} From cbaf65cbf36443b016036c089df9f02c9fbe89d2 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sat, 27 Sep 2025 08:02:20 -0400 Subject: [PATCH 3/9] DuckDBResult: Add aysncIterator for DataChunks with helper methods Adds an asyncIterator to the DuckDBResult class to return the raw DuckDBDataChunks. Helper methods to perform common operations such as returning row objects or returning the rows as JSON have been added here so that the caller can iterate over the results in a way that's convenient for them. --- api/src/DuckDBResult.ts | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index 2f749fb3..70fbcf6a 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -239,4 +239,56 @@ export class DuckDBResult { public async getRowObjectsJson(): Promise[]> { return this.convertRowObjects(JsonDuckDBValueConverter); } + + public async *[Symbol.asyncIterator](): AsyncIterableIterator { + while (true) { + const chunk = await this.fetchChunk(); + if (chunk && chunk.rowCount > 0) { + yield chunk; + } else { + break; + } + } + } + + public async *yieldRows(): AsyncIterableIterator { + const iterator = this[Symbol.asyncIterator](); + for await (const chunk of iterator) { + yield getRowsFromChunks([chunk]); + } + } + + public async *yieldRowObjects(): AsyncIterableIterator< + Record[] + > { + const iterator = this[Symbol.asyncIterator](); + const deduplicatedColumnNames = this.deduplicatedColumnNames(); + + for await (const chunk of iterator) { + yield getRowObjectsFromChunks([chunk], deduplicatedColumnNames); + } + } + + public async *yieldConvertedRows( + converter: DuckDBValueConverter, + ): AsyncIterableIterator[]> { + const iterator = this[Symbol.asyncIterator](); + const deduplicatedColumnNames = this.deduplicatedColumnNames(); + + for await (const chunk of iterator) { + yield convertRowObjectsFromChunks( + [chunk], + deduplicatedColumnNames, + converter, + ); + } + } + + public async *yieldRowsJs(): AsyncIterableIterator[]> { + return this.convertRowObjects(JSDuckDBValueConverter); + } + + public async *yieldRowsJson(): AsyncIterableIterator[]> { + return this.convertRowObjects(JsonDuckDBValueConverter); + } } From 473613d83934ffdb664ecbe8a8e861eaf831e6e3 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 09:26:45 -0400 Subject: [PATCH 4/9] DuckDBResult: Add `yieldConvertedRowObjects` `yieldConvertedRows` has been refactored to use `convertRowsFromChunks`. `yeidlConvertedRowObjects` now uses `convertRowObjectsFromChunks` as the previous version of `yieldConvertedRows` did. --- api/src/DuckDBResult.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index 70fbcf6a..7e0c6941 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -270,12 +270,18 @@ export class DuckDBResult { } public async *yieldConvertedRows( - converter: DuckDBValueConverter, + converter: DuckDBValueConverter + ): AsyncIterableIterator<(T | null)[][]> { + for await (const chunk of this) { + yield convertRowsFromChunks([chunk], converter); + } + } + + public async *yieldConvertedRowObjects( + converter: DuckDBValueConverter ): AsyncIterableIterator[]> { - const iterator = this[Symbol.asyncIterator](); const deduplicatedColumnNames = this.deduplicatedColumnNames(); - - for await (const chunk of iterator) { + for await (const chunk of this) { yield convertRowObjectsFromChunks( [chunk], deduplicatedColumnNames, From a8a239cbe98bef98230f849b6b8838624bc13317 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 09:28:33 -0400 Subject: [PATCH 5/9] DuckDBResult: Use `this` instead of defining `iterator` variable Converts the remaining declarations of `iterator` to just use `this` directly. --- api/src/DuckDBResult.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index 7e0c6941..2578d950 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -252,8 +252,7 @@ export class DuckDBResult { } public async *yieldRows(): AsyncIterableIterator { - const iterator = this[Symbol.asyncIterator](); - for await (const chunk of iterator) { + for await (const chunk of this) { yield getRowsFromChunks([chunk]); } } @@ -261,10 +260,8 @@ export class DuckDBResult { public async *yieldRowObjects(): AsyncIterableIterator< Record[] > { - const iterator = this[Symbol.asyncIterator](); const deduplicatedColumnNames = this.deduplicatedColumnNames(); - - for await (const chunk of iterator) { + for await (const chunk of this) { yield getRowObjectsFromChunks([chunk], deduplicatedColumnNames); } } From 253e84e02013d08e0614a037dca8acb56e2a51b0 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 09:29:37 -0400 Subject: [PATCH 6/9] DuckDBResult: Fix `yieldRows*` methods and add `yieldRowObject*` methods These were a bit backwards. `yieldRowsJs` and `yieldRowsJson` now call `yieldConvertedRows`. The object versions of these methods have been added here as `yieldRowObjectJs` and `yieldRowObjectJson` which both call `yieldConvertedRowObjects`. --- api/src/DuckDBResult.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/api/src/DuckDBResult.ts b/api/src/DuckDBResult.ts index 2578d950..52c5c892 100644 --- a/api/src/DuckDBResult.ts +++ b/api/src/DuckDBResult.ts @@ -287,11 +287,23 @@ export class DuckDBResult { } } - public async *yieldRowsJs(): AsyncIterableIterator[]> { - return this.convertRowObjects(JSDuckDBValueConverter); + public yieldRowsJs(): AsyncIterableIterator { + return this.yieldConvertedRows(JSDuckDBValueConverter); } - public async *yieldRowsJson(): AsyncIterableIterator[]> { - return this.convertRowObjects(JsonDuckDBValueConverter); + public yieldRowsJson(): AsyncIterableIterator { + return this.yieldConvertedRows(JsonDuckDBValueConverter); + } + + public yieldRowObjectJs(): AsyncIterableIterator< + Record[] + > { + return this.yieldConvertedRowObjects(JSDuckDBValueConverter); + } + + public yieldRowObjectJson(): AsyncIterableIterator< + Record[] + > { + return this.yieldConvertedRowObjects(JsonDuckDBValueConverter); } } From 6e414884646ef09c79ffbe66af197af0c9062eb3 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 09:42:17 -0400 Subject: [PATCH 7/9] api.test: Add test for async stream iteration The first test added is to ensure that the caller can iterate over the DuckDBResult object to get the data chunks. The last 4 tests added here test the `yieldConvertedRows` and `yieldConvertedRowObjects` along with the `yieldRows*` and `yieldRowObject*` methods since the latter methods call the former methods. --- api/test/api.test.ts | 67 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/api/test/api.test.ts b/api/test/api.test.ts index 9accbb98..f57a8341 100644 --- a/api/test/api.test.ts +++ b/api/test/api.test.ts @@ -2445,4 +2445,71 @@ ORDER BY name assert.equal(quotedIdentifier('table name'), '"table name"'); }); }); + + test("iterate over DuckDBResult stream in chunks", async () => { + await withConnection(async (connection) => { + const result = await connection.stream( + "select i::int, i::int + 10, (i + 100)::varchar from range(3) t(i)", + ); + + for await (const chunk of result) { + assert.strictEqual(chunk.rowCount, 3); + let i = 0; + assertValues( + chunk, + i++, + DuckDBIntegerVector, + [0, 1, 2], + ); + assertValues( + chunk, + i++, + DuckDBIntegerVector, + [10, 11, 12], + ); + assertValues( + chunk, + i++, + DuckDBVarCharVector, + ["100", "101", "102"], + ); + } + }); + }); + + test("iterate result stream rows js", async () => { + await withConnection(async (connection) => { + const result = await connection.stream(createTestJSQuery()); + for await (const row of result.yieldRowsJs()) { + assert.deepEqual(row, createTestJSRowsJS()); + } + }); + }); + + test("iterate result stream object js", async () => { + await withConnection(async (connection) => { + const result = await connection.stream(createTestJSQuery()); + for await (const row of result.yieldRowObjectJs()) { + assert.deepEqual(row, createTestJSRowObjectsJS()); + } + }); + }); + + test("iterate result stream rows json", async () => { + await withConnection(async (connection) => { + const result = await connection.stream(`from test_all_types()`); + for await (const row of result.yieldRowsJson()) { + assert.deepEqual(row, createTestAllTypesRowsJson()); + } + }); + }); + + test("iterate result stream object json", async () => { + await withConnection(async (connection) => { + const result = await connection.stream(`from test_all_types()`); + for await (const row of result.yieldRowObjectJson()) { + assert.deepEqual(row, createTestAllTypesRowObjectsJson()); + } + }); + }); }); From 396c1f158b8b07437298aac1eee10b118396f173 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 13:07:17 -0400 Subject: [PATCH 8/9] api.test: Add test case that tests generating more than one chunk --- api/test/api.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/api/test/api.test.ts b/api/test/api.test.ts index f57a8341..32e638fd 100644 --- a/api/test/api.test.ts +++ b/api/test/api.test.ts @@ -2477,6 +2477,24 @@ ORDER BY name }); }); + test("iterate over many DuckDBResult chunks", async () => { + await withConnection(async (connection) => { + const chunkSize = 2048; + const totalExpectedCount = chunkSize * 3; + const result = await connection.stream( + `select i::int from range(${totalExpectedCount}) t(i)` + ); + + let total = 0; + for await (const chunk of result) { + assert.equal(chunk.rowCount, chunkSize); + total += chunk.rowCount; + } + + assert.equal(total, totalExpectedCount); + }); + }); + test("iterate result stream rows js", async () => { await withConnection(async (connection) => { const result = await connection.stream(createTestJSQuery()); From feadcbe4146ddbe3be412ef26d32585c9f14d203 Mon Sep 17 00:00:00 2001 From: Robert Stefanic Date: Sun, 28 Sep 2025 13:17:34 -0400 Subject: [PATCH 9/9] api.test: Add tests for `yieldRows` and `yieldRowObjects` --- api/test/api.test.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/api/test/api.test.ts b/api/test/api.test.ts index 32e638fd..7e4b9533 100644 --- a/api/test/api.test.ts +++ b/api/test/api.test.ts @@ -2495,6 +2495,46 @@ ORDER BY name }); }); + test("iterate stream of rows", async () => { + await withConnection(async (connection) => { + const result = await connection.stream( + "select i::int, i::int + 10, (i + 100)::varchar from range(3) t(i)", + ); + + const expectedRows: DuckDBValue[][] = [ + [0, 10, '100'], + [1, 11, '101'], + [2, 12, '102'] + ]; + + for await (const rows of result.yieldRows()) { + for (let i = 0; i < rows.length; i++) { + assert.deepEqual(rows[i], expectedRows[i]); + } + } + }); + }); + + test("iterate stream of row objects", async () => { + await withConnection(async (connection) => { + const result = await connection.stream( + "select i::int as a, i::int + 10 as b, (i + 100)::varchar as c from range(3) t(i)", + ); + + const expectedRows: Record[] = [ + { a: 0, b: 10, c: '100'}, + { a: 1, b: 11, c: '101'}, + { a: 2, b: 12, c: '102'} + ]; + + for await (const rows of result.yieldRowObjects()) { + for (let i = 0; i < rows.length; i++) { + assert.deepEqual(rows[i], expectedRows[i]); + } + } + }); + }); + test("iterate result stream rows js", async () => { await withConnection(async (connection) => { const result = await connection.stream(createTestJSQuery());