diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index 5ea46d86..aa9de768 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -347,6 +347,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -362,6 +365,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { @@ -387,6 +391,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -439,6 +444,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -456,6 +462,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -650,6 +657,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -665,6 +675,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { @@ -690,6 +701,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -742,6 +754,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -759,6 +772,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -971,6 +985,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -983,6 +1000,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1004,6 +1022,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1047,6 +1066,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1064,6 +1084,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1276,6 +1297,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1291,6 +1315,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { @@ -1316,6 +1341,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1368,6 +1394,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1385,6 +1412,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1578,6 +1606,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1590,6 +1621,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1611,6 +1643,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1654,6 +1687,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1671,6 +1705,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1851,6 +1886,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1863,6 +1901,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1884,6 +1923,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1927,6 +1967,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1944,6 +1985,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -2135,6 +2177,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2150,6 +2195,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { @@ -2175,6 +2221,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2227,6 +2274,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -2244,6 +2292,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -2362,6 +2411,9 @@ export declare const components: { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2374,6 +2426,7 @@ export declare const components: { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -2395,6 +2448,7 @@ export declare const components: { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -2438,6 +2492,7 @@ export declare const components: { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -2455,6 +2510,7 @@ export declare const components: { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; diff --git a/src/UIMessages.ts b/src/UIMessages.ts index 8e6fddaa..7a3d15ba 100644 --- a/src/UIMessages.ts +++ b/src/UIMessages.ts @@ -381,10 +381,9 @@ function createAssistantUIMessage< const allParts: UIMessage["parts"] = []; for (const message of group) { - const coreMessage = message.message && toModelMessage(message.message); - if (!coreMessage) continue; + if (!message.message) continue; - const content = coreMessage.content; + const content = message.message.content; const nonStringContent = content && typeof content !== "string" ? content : []; const text = extractTextFromMessageDoc(message); @@ -441,12 +440,11 @@ function createAssistantUIMessage< type: "step-start", } satisfies StepStartUIPart); const toolPart: ToolUIPart = { + ...omit(contentPart, ["args", "type"]), type: `tool-${contentPart.toolName as keyof TOOLS & string}`, - toolCallId: contentPart.toolCallId, - input: contentPart.input as DeepPartial< + input: contentPart.args as DeepPartial< TOOLS[keyof TOOLS & string]["input"] >, - providerExecuted: contentPart.providerExecuted, ...(message.streaming ? { state: "input-streaming" } : { @@ -459,9 +457,9 @@ function createAssistantUIMessage< } case "tool-result": { const output = - contentPart.output?.type === "json" + typeof contentPart.output?.type === "string" ? contentPart.output.value - : contentPart.output; + : (contentPart.output ?? contentPart.result); const call = allParts.find( (part) => part.type === `tool-${contentPart.toolName}` && @@ -478,25 +476,22 @@ function createAssistantUIMessage< call.output = output; } } else { - console.warn( - "Tool result without preceding tool call.. adding anyways", - contentPart, - ); if (message.error) { allParts.push({ + input: contentPart.args, + ...omit(contentPart, ["args", "type", "output"]), type: `tool-${contentPart.toolName}`, - toolCallId: contentPart.toolCallId, state: "output-error", - input: undefined, + output, errorText: message.error, callProviderMetadata: message.providerMetadata, } satisfies ToolUIPart); } else { allParts.push({ + input: contentPart.args, + ...omit(contentPart, ["args", "type", "output"]), type: `tool-${contentPart.toolName}`, - toolCallId: contentPart.toolCallId, state: "output-available", - input: undefined, output, callProviderMetadata: message.providerMetadata, } satisfies ToolUIPart); diff --git a/src/client/index.ts b/src/client/index.ts index 076cbcf1..682d7bae 100644 --- a/src/client/index.ts +++ b/src/client/index.ts @@ -1,7 +1,8 @@ -import type { - FlexibleSchema, - IdGenerator, - InferSchema, +import { + getErrorMessage, + type FlexibleSchema, + type IdGenerator, + type InferSchema, } from "@ai-sdk/provider-utils"; import type { CallSettings, @@ -537,7 +538,7 @@ export class Agent< }; return Object.assign(result, metadata); } catch (error) { - await call.fail(errorToString(error)); + await call.fail(getErrorMessage(error)); throw error; } } @@ -642,8 +643,8 @@ export class Agent< ), onError: async (error) => { console.error("onError", error); - await call.fail(errorToString(error.error)); - await streamer?.fail(errorToString(error.error)); + await call.fail(getErrorMessage(error.error)); + await streamer?.fail(getErrorMessage(error.error)); return streamTextArgs.onError?.(error); }, prepareStep: async (options) => { @@ -743,7 +744,7 @@ export class Agent< }; return Object.assign(result, metadata); } catch (error) { - await fail(errorToString(error)); + await fail(getErrorMessage(error)); throw error; } } @@ -797,7 +798,7 @@ export class Agent< console.error(" streamObject onError", error); // TODO: content that we have so far // content: stream.fullStream. - await fail(errorToString(error.error)); + await fail(getErrorMessage(error.error)); return args.onError?.(error); }, onFinish: async (result) => { @@ -1668,10 +1669,3 @@ async function willContinue( } return !!stopWhen && !(await stopWhen({ steps })); } - -function errorToString(error: unknown): string { - if (error instanceof Error) { - return error.message; - } - return String(error); -} diff --git a/src/client/streaming.ts b/src/client/streaming.ts index 23449a47..d72f0159 100644 --- a/src/client/streaming.ts +++ b/src/client/streaming.ts @@ -24,6 +24,7 @@ import type { RunQueryCtx, SyncStreamsReturnValue, } from "./types.js"; +import { getErrorMessage } from "@ai-sdk/provider-utils"; export const vStreamMessagesReturnValue = v.object({ ...vPaginationResult(vMessageDoc).fields, @@ -312,9 +313,7 @@ export class DeltaStreamer { return; } } catch (e) { - await this.config.onAsyncAbort( - e instanceof Error ? e.message : "unknown error", - ); + await this.config.onAsyncAbort(getErrorMessage(e)); this.abortController.abort(); throw e; } diff --git a/src/component/_generated/api.d.ts b/src/component/_generated/api.d.ts index 3257e2f7..7cb89b2a 100644 --- a/src/component/_generated/api.d.ts +++ b/src/component/_generated/api.d.ts @@ -232,6 +232,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -244,6 +247,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -265,6 +269,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -308,6 +313,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -325,6 +331,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -494,6 +501,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -506,6 +516,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -527,6 +538,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -570,6 +582,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -587,6 +600,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -777,6 +791,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -786,6 +803,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -807,6 +825,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -841,6 +860,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -858,6 +878,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1049,6 +1070,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1061,6 +1085,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1082,6 +1107,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1125,6 +1151,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1142,6 +1169,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1313,6 +1341,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1322,6 +1353,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1343,6 +1375,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1377,6 +1410,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1394,6 +1428,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1556,6 +1591,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1565,6 +1603,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1586,6 +1625,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1620,6 +1660,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1637,6 +1678,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -1807,6 +1849,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1819,6 +1864,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1840,6 +1886,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record< string, @@ -1883,6 +1930,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -1900,6 +1948,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -2000,6 +2049,9 @@ export type Mounts = { } | { args: any; + dynamic?: boolean; + error?: any; + invalid?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -2009,6 +2061,7 @@ export type Mounts = { } | { args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -2030,6 +2083,7 @@ export type Mounts = { } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; @@ -2064,6 +2118,7 @@ export type Mounts = { | { content: Array<{ args?: any; + dynamic?: boolean; experimental_content?: Array< | { text: string; type: "text" } | { data: string; mimeType?: string; type: "image" } @@ -2081,6 +2136,7 @@ export type Mounts = { | { data: string; mediaType: string; type: "media" } >; }; + preliminary?: boolean; providerExecuted?: boolean; providerMetadata?: Record>; providerOptions?: Record>; diff --git a/src/mapping.ts b/src/mapping.ts index b37264b4..de674113 100644 --- a/src/mapping.ts +++ b/src/mapping.ts @@ -35,6 +35,8 @@ import { type vToolResultPart, type SourcePart, vToolResultOutput, + vAssistantContent, + vToolContent, } from "./validators.js"; import type { ActionCtx, AgentComponent } from "./client/types.js"; import type { RunMutationCtx } from "./client/types.js"; @@ -45,12 +47,15 @@ import { type ProviderOptions, type ReasoningPart, } from "@ai-sdk/provider-utils"; +import { type LanguageModelV2ToolResultOutput } from "@ai-sdk/provider"; import { parse, validate } from "convex-helpers/validators"; import { + extractText, getModelName, getProviderName, type ModelOrMetadata, } from "./shared.js"; +import { omit, pick } from "convex-helpers"; export type AIMessageWithoutId = Omit; export type SerializeUrlsAndUint8Arrays = T extends URL @@ -98,9 +103,7 @@ export function fromModelMessage(message: ModelMessage): Message { return { role: message.role, content, - ...(message.providerOptions - ? { providerOptions: message.providerOptions } - : {}), + ...pick(message, ["providerOptions"]), } as SerializedMessage; } @@ -117,9 +120,7 @@ export async function serializeOrThrow( return { role: message.role, content, - ...(message.providerOptions - ? { providerOptions: message.providerOptions } - : {}), + ...pick(message, ["providerOptions"]), } as SerializedMessage; } @@ -179,38 +180,67 @@ export async function serializeNewMessagesInStep( step: StepResult, model: ModelOrMetadata | undefined, ): Promise<{ messages: MessageWithMetadata[] }> { - // If there are tool results, there's another message with the tool results - // ref: https://github.com/vercel/ai/blob/main/packages/ai/core/generate-text/to-response-messages.ts - const assistantFields = { - model: model ? getModelName(model) : undefined, - provider: model ? getProviderName(model) : undefined, - providerMetadata: step.providerMetadata, - reasoning: step.reasoningText, - reasoningDetails: step.reasoning, - usage: serializeUsage(step.usage), - warnings: serializeWarnings(step.warnings), - finishReason: step.finishReason, - // Only store the sources on one message - sources: step.toolResults.length === 0 ? step.sources : undefined, - } satisfies Omit; - const toolFields = { sources: step.sources }; - const messages: MessageWithMetadata[] = await Promise.all( - (step.toolResults.length > 0 - ? step.response.messages.slice(-2) - : step.content.length - ? step.response.messages.slice(-1) - : [{ role: "assistant" as const, content: [] }] - ).map(async (msg): Promise => { - const { message, fileIds } = await serializeMessage(ctx, component, msg); - return parse(vMessageWithMetadata, { - message, - ...(message.role === "tool" ? toolFields : assistantFields), - text: step.text, - fileIds, - }); - }), + const toolResultIndex = step.content.findIndex( + (c) => + (c.type === "tool-result" || c.type === "tool-error") && + !c.providerExecuted, + ); + const hasToolResults = toolResultIndex !== -1; + const assistantContent = hasToolResults + ? step.content.slice(0, toolResultIndex) + : step.content; + const { content, fileIds } = await serializeStepContent( + ctx, + component, + assistantContent, ); - // TODO: capture step.files separately? + const message = { + role: "assistant" as const, + content: content as Infer, + providerOptions: (hasToolResults + ? step.response.messages.at(-2) + : step.response.messages.at(-1) + )?.providerOptions, + } satisfies Message; + const messages = [ + parse(vMessageWithMetadata, { + model: model ? getModelName(model) : undefined, + provider: model ? getProviderName(model) : undefined, + providerMetadata: step.providerMetadata, + reasoning: step.reasoningText, + reasoningDetails: step.reasoning, + usage: serializeUsage(step.usage), + warnings: serializeWarnings(step.warnings), + finishReason: step.finishReason, + fileIds, + // Only store the sources on one message + sources: assistantContent.filter((c) => c.type === "source"), + message, + text: extractText(message) || step.text, + }), + ]; + + if (hasToolResults) { + const toolContent = step.content.slice(toolResultIndex); + const { content, fileIds } = await serializeStepContent( + ctx, + component, + toolContent, + ); + const toolMessage = { + role: "tool" as const, + content: content as Infer, + providerOptions: step.response.messages.at(-1)?.providerOptions, + } satisfies Message; + messages.push( + parse(vMessageWithMetadata, { + message: toolMessage, + sources: toolContent.filter((c) => c.type === "source"), + fileIds, + finishReason: step.finishReason, + }), + ); + } return { messages }; } @@ -253,6 +283,129 @@ function getMimeOrMediaType(part: { mediaType?: string; mimeType?: string }) { return undefined; } +export async function serializeStepContent( + ctx: ActionCtx, + component: AgentComponent, + content: StepResult["content"], +): Promise<{ content: SerializedContent; fileIds?: string[] }> { + const fileIds: string[] = []; + const serialized = await Promise.all( + content.map(async (part) => { + const metadata: { + providerOptions?: ProviderOptions; + providerMetadata?: ProviderMetadata; + } = {}; + if ("providerOptions" in part) { + metadata.providerOptions = part.providerOptions as ProviderOptions; + } else if ("providerMetadata" in part) { + metadata.providerMetadata = part.providerMetadata; + } + switch (part.type) { + case "text": { + return part satisfies Infer; + } + case "reasoning": + return part satisfies Infer; + case "source": + return part satisfies Infer; + case "tool-call": { + return { + ...pick(part, [ + "type", + "toolCallId", + "toolName", + "providerExecuted", + "dynamic", + "error", + "invalid", + ]), + args: part.input, + ...metadata, + } satisfies Infer; + } + case "tool-result": { + const rawOutput = part.output; + const output: LanguageModelV2ToolResultOutput = + typeof rawOutput === "string" + ? { type: "text", value: rawOutput } + : { type: "json", value: part.output ?? null }; + + return { + ...pick(part, [ + "type", + "toolCallId", + "toolName", + "dynamic", + "preliminary", + "providerExecuted", + ]), + args: part.input, + output, + ...metadata, + } satisfies Infer; + } + case "tool-error": + return { + ...pick(part, [ + "toolCallId", + "toolName", + "dynamic", + "providerExecuted", + ]), + type: "tool-result", + args: part.input, + output: + part.error instanceof Error + ? { + type: "error-text", + value: part.error.message, + } + : typeof part.error === "string" + ? { + type: "error-text", + value: part.error, + } + : { + type: "error-json", + value: part.error ?? null, + }, + ...metadata, + } satisfies Infer; + case "file": { + const uint8Array = part.file.uint8Array; + let data: ArrayBuffer | string = uint8Array.buffer.slice( + uint8Array.byteOffset, + uint8Array.byteOffset + uint8Array.byteLength, + ) as ArrayBuffer; + const mimeType = getMimeOrMediaType(part.file)!; + if (data.byteLength > MAX_FILE_SIZE) { + const { file } = await storeFile( + ctx, + component, + new Blob([data], { type: mimeType }), + ); + data = file.url; + fileIds.push(file.fileId); + } + return { + type: part.type, + data, + mimeType, + ...metadata, + } satisfies Infer; + } + + default: + return part satisfies Infer; + } + }), + ); + return { + content: serialized as SerializedContent, + fileIds: fileIds.length > 0 ? fileIds : undefined, + }; +} + export async function serializeContent( ctx: ActionCtx | RunMutationCtx, component: AgentComponent, @@ -463,20 +616,20 @@ export function toModelMessageContent( mediaType: getMimeOrMediaType(part)!, ...metadata, } satisfies FilePart; - case "tool-call": { - const input = "input" in part ? part.input : part.args; + case "tool-call": return { - type: part.type, - input: input ?? null, - toolCallId: part.toolCallId, - toolName: part.toolName, - providerExecuted: part.providerExecuted, - ...metadata, + input: ("input" in part ? part.input : part.args) ?? null, + ...omit(part as Infer, ["args"]), } satisfies ToolCallPart; - } - case "tool-result": { - return normalizeToolResult(part, metadata); - } + case "tool-result": + return { + input: (part as Infer).args, + output: normalizeToolOutput( + (part as Infer).result, + ), + ...omit(part as Infer, ["result", "args"]), + ...metadata, + } satisfies ToolResultPart & { input: unknown }; case "reasoning": return { type: part.type, @@ -534,16 +687,13 @@ function normalizeToolResult( providerOptions?: ProviderOptions; providerMetadata?: ProviderMetadata; }, -): ToolResultPart & Infer { +): ToolResultPart & Infer & { input: unknown } { return { - type: part.type, - output: - part.output ?? - normalizeToolOutput("result" in part ? part.result : undefined), - toolCallId: part.toolCallId, - toolName: part.toolName, + input: (part as Infer).args, + output: normalizeToolOutput("result" in part ? part.result : undefined), + ...omit(part as Infer, ["result", "args"]), ...metadata, - } satisfies ToolResultPart; + } satisfies ToolResultPart & { input: unknown }; } /** @@ -654,37 +804,26 @@ export function toModelMessageDataOrUrl( return urlOrString; } -export function toUIFilePart(part: ImagePart | FilePart): FileUIPart { +export function toUIFilePart( + part: + | ImagePart + | FilePart + | Infer + | Infer, +): FileUIPart { const dataOrUrl = part.type === "image" ? part.image : part.data; const url = dataOrUrl instanceof ArrayBuffer ? convertUint8ArrayToBase64(new Uint8Array(dataOrUrl)) : dataOrUrl.toString(); + const mediaType = getMimeOrMediaType(part); + return { type: "file", - mediaType: part.mediaType!, + mediaType: mediaType!, filename: part.type === "file" ? part.filename : undefined, url, providerMetadata: part.providerOptions, }; } - -// Currently unused -// export function toModelMessages(args: { -// messages?: ModelMessage[] | AIMessageWithoutId[]; -// }): ModelMessage[] { -// const messages: ModelMessage[] = []; -// if (args.messages) { -// if ( -// args.messages.every( -// (m) => typeof m === "object" && m !== null && "parts" in m, -// ) -// ) { -// messages.push(...convertToModelMessages(args.messages)); -// } else { -// messages.push(...modelMessageSchema.array().parse(args.messages)); -// } -// } -// return messages; -// } diff --git a/src/toUIMessages.test.ts b/src/toUIMessages.test.ts index 8648a7b4..5dfc8452 100644 --- a/src/toUIMessages.test.ts +++ b/src/toUIMessages.test.ts @@ -132,10 +132,7 @@ describe("toUIMessages", () => { type: "tool-myTool", toolCallId: "call1", state: "output-available", - output: { - type: "text", - value: "42", - }, + output: "42", }); }); @@ -252,7 +249,7 @@ describe("toUIMessages", () => { type: "tool-result", toolName: "myTool", toolCallId: "call1", - result: { + output: { type: "json", value: { data: "wrapped result", success: true }, }, @@ -648,10 +645,7 @@ describe("toUIMessages", () => { toolCallId: "call1", state: "output-available", input: { operation: "add", a: 40, b: 2 }, - output: { - type: "text", - value: "42", - }, + output: "42", }); // Should also have text part diff --git a/src/validators.ts b/src/validators.ts index bac67a1b..a152af37 100644 --- a/src/validators.ts +++ b/src/validators.ts @@ -114,8 +114,11 @@ export const vToolCallPart = v.object({ type: v.literal("tool-call"), toolCallId: v.string(), toolName: v.string(), - args: v.any(), + args: v.any(), // referred to as input in ai v5 providerExecuted: v.optional(v.boolean()), + dynamic: v.optional(v.boolean()), + invalid: v.optional(v.boolean()), + error: v.optional(v.any()), providerOptions, providerMetadata, }); @@ -159,14 +162,18 @@ export const vToolResultPart = v.object({ providerOptions, providerMetadata, + // In a modelMessage this is redundant with the call & stripped + // Referred to as input in ai v5 + args: v.optional(v.any()), providerExecuted: v.optional(v.boolean()), + dynamic: v.optional(v.boolean()), + preliminary: v.optional(v.boolean()), // Deprecated in ai v5 result: v.optional(v.any()), // either this or output will be present isError: v.optional(v.boolean()), // This is only here b/c steps include it in toolResults // Normal ModelMessage doesn't have this - args: v.optional(v.any()), experimental_content: v.optional(vToolResultContent), }); export const vToolContent = v.array(vToolResultPart);