Update M.E.AI.OpenAI for latest OpenAI release (#6577)

Author: stephentoub

eng/packages/General.props

@@ -16,7 +16,7 @@
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="OllamaSharp" Version="5.1.9" />
-    <PackageVersion Include="OpenAI" Version="2.2.0-beta.4" />
+    <PackageVersion Include="OpenAI" Version="2.2.0" />
     <PackageVersion Include="Polly" Version="8.4.2" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />
     <PackageVersion Include="Polly.Extensions" Version="8.4.2" />

src/Libraries/Microsoft.Extensions.AI.Abstractions/SpeechToText/SpeechToTextOptions.cs

@@ -11,20 +11,20 @@ namespace Microsoft.Extensions.AI;
 [Experimental("MEAI001")]
 public class SpeechToTextOptions
 {
+    /// <summary>Gets or sets any additional properties associated with the options.</summary>
+    public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
+
     /// <summary>Gets or sets the model ID for the speech to text.</summary>
     public string? ModelId { get; set; }
 
     /// <summary>Gets or sets the language of source speech.</summary>
     public string? SpeechLanguage { get; set; }
 
-    /// <summary>Gets or sets the language for the target generated text.</summary>
-    public string? TextLanguage { get; set; }
-
     /// <summary>Gets or sets the sample rate of the speech input audio.</summary>
     public int? SpeechSampleRate { get; set; }
 
-    /// <summary>Gets or sets any additional properties associated with the options.</summary>
-    public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
+    /// <summary>Gets or sets the language for the target generated text.</summary>
+    public string? TextLanguage { get; set; }
 
     /// <summary>
     /// Gets or sets a callback responsible for creating the raw representation of the embedding generation options from an underlying implementation.
@@ -51,11 +51,11 @@ public virtual SpeechToTextOptions Clone()
     {
         SpeechToTextOptions options = new()
         {
+            AdditionalProperties = AdditionalProperties?.Clone(),
             ModelId = ModelId,
             SpeechLanguage = SpeechLanguage,
-            TextLanguage = TextLanguage,
             SpeechSampleRate = SpeechSampleRate,
-            AdditionalProperties = AdditionalProperties?.Clone(),
+            TextLanguage = TextLanguage,
         };
 
         return options;

src/Libraries/Microsoft.Extensions.AI.OpenAI/Microsoft.Extensions.AI.OpenAI.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <RootNamespace>Microsoft.Extensions.AI</RootNamespace>
@@ -15,8 +15,8 @@
 
   <PropertyGroup>
     <TargetFrameworks>$(TargetFrameworks);netstandard2.0</TargetFrameworks>
-    <NoWarn>$(NoWarn);CA1063;CA1508;CA2227;SA1316;S1121;S3358;EA0002;OPENAI002</NoWarn>
-    <NoWarn>$(NoWarn);MEAI001</NoWarn>
+    <NoWarn>$(NoWarn);CA1063;CA1508;CA2227;SA1316;S1121;S3358;EA0002</NoWarn>
+    <NoWarn>$(NoWarn);OPENAI001;OPENAI002;MEAI001</NoWarn>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <DisableNETStandardCompatErrors>true</DisableNETStandardCompatErrors>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantChatClient.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
@@ -28,7 +27,6 @@
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents an <see cref="IChatClient"/> for an Azure.AI.Agents.Persistent <see cref="AssistantClient"/>.</summary>
-[Experimental("OPENAI001")]
 internal sealed class OpenAIAssistantChatClient : IChatClient
 {
     /// <summary>The underlying <see cref="AssistantClient" />.</summary>

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs

@@ -91,7 +91,7 @@ public IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
         // Make the call to OpenAI.
         var chatCompletionUpdates = _chatClient.CompleteChatStreamingAsync(openAIChatMessages, openAIOptions, cancellationToken);
 
-        return FromOpenAIStreamingChatCompletionAsync(chatCompletionUpdates, cancellationToken);
+        return FromOpenAIStreamingChatCompletionAsync(chatCompletionUpdates, openAIOptions, cancellationToken);
     }
 
     /// <inheritdoc />
@@ -290,7 +290,8 @@ private static List<ChatMessageContentPart> ToOpenAIChatContent(IList<AIContent>
 
     private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingChatCompletionAsync(
         IAsyncEnumerable<StreamingChatCompletionUpdate> updates,
-        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+        ChatCompletionOptions? options,
+        [EnumeratorCancellation] CancellationToken cancellationToken)
     {
         Dictionary<int, FunctionCallInfo>? functionCallInfos = null;
         ChatRole? streamedRole = null;
@@ -334,6 +335,14 @@ private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingCha
                 }
             }
 
+            if (update.OutputAudioUpdate is { } audioUpdate)
+            {
+                responseUpdate.Contents.Add(new DataContent(audioUpdate.AudioBytesUpdate.ToMemory(), GetOutputAudioMimeType(options))
+                {
+                    RawRepresentation = audioUpdate,
+                });
+            }
+
             // Transfer over refusal updates.
             if (update.RefusalUpdate is not null)
             {
@@ -363,8 +372,10 @@ private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingCha
             // Transfer over usage updates.
             if (update.Usage is ChatTokenUsage tokenUsage)
             {
-                var usageDetails = FromOpenAIUsage(tokenUsage);
-                responseUpdate.Contents.Add(new UsageContent(usageDetails));
+                responseUpdate.Contents.Add(new UsageContent(FromOpenAIUsage(tokenUsage))
+                {
+                    RawRepresentation = tokenUsage,
+                });
             }
 
             // Now yield the item.
@@ -408,6 +419,17 @@ private static async IAsyncEnumerable<ChatResponseUpdate> FromOpenAIStreamingCha
         }
     }
 
+    private static string GetOutputAudioMimeType(ChatCompletionOptions? options) =>
+        options?.AudioOptions?.OutputAudioFormat.ToString()?.ToLowerInvariant() switch
+        {
+            "opus" => "audio/opus",
+            "aac" => "audio/aac",
+            "flac" => "audio/flac",
+            "wav" => "audio/wav",
+            "pcm" => "audio/pcm",
+            "mp3" or _ => "audio/mpeg",
+        };
+
     private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAICompletion, ChatOptions? options, ChatCompletionOptions chatCompletionOptions)
     {
         _ = Throw.IfNull(openAICompletion);
@@ -432,19 +454,10 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple
         // Output audio is handled separately from message content parts.
         if (openAICompletion.OutputAudio is ChatOutputAudio audio)
         {
-            string mimeType = chatCompletionOptions?.AudioOptions?.OutputAudioFormat.ToString()?.ToLowerInvariant() switch
+            returnMessage.Contents.Add(new DataContent(audio.AudioBytes.ToMemory(), GetOutputAudioMimeType(chatCompletionOptions))
             {
-                "opus" => "audio/opus",
-                "aac" => "audio/aac",
-                "flac" => "audio/flac",
-                "wav" => "audio/wav",
-                "pcm" => "audio/pcm",
-                "mp3" or _ => "audio/mpeg",
-            };
-
-            var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType);
-
-            returnMessage.Contents.Add(dc);
+                RawRepresentation = audio,
+            });
         }
 
         // Also manufacture function calling content items from any tool calls in the response.
@@ -505,9 +518,7 @@ private ChatCompletionOptions ToOpenAIOptions(ChatOptions? options)
         result.PresencePenalty ??= options.PresencePenalty;
         result.Temperature ??= options.Temperature;
         result.AllowParallelToolCalls ??= options.AllowMultipleToolCalls;
-#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates.
         result.Seed ??= options.Seed;
-#pragma warning restore OPENAI001
 
         if (options.StopSequences is { Count: > 0 } stopSequences)
         {

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIClientExtensions.cs

@@ -14,7 +14,7 @@
 using OpenAI.Audio;
 using OpenAI.Chat;
 using OpenAI.Embeddings;
-using OpenAI.RealtimeConversation;
+using OpenAI.Realtime;
 using OpenAI.Responses;
 
 #pragma warning disable S103 // Lines should not be too long
@@ -134,7 +134,6 @@ public static IChatClient AsIChatClient(this OpenAIResponseClient responseClient
     /// <exception cref="ArgumentNullException"><paramref name="assistantClient"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentNullException"><paramref name="assistantId"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentException"><paramref name="assistantId"/> is empty or composed entirely of whitespace.</exception>
-    [Experimental("OPENAI001")] // AssistantClient itself is experimental with this ID
     public static IChatClient AsIChatClient(this AssistantClient assistantClient, string assistantId, string? threadId = null) =>
         new OpenAIAssistantChatClient(assistantClient, assistantId, threadId);
 
@@ -165,7 +164,6 @@ public static ChatTool AsOpenAIChatTool(this AIFunction function) =>
     /// <param name="function">The function to convert.</param>
     /// <returns>An OpenAI <see cref="FunctionToolDefinition"/> representing <paramref name="function"/>.</returns>
     /// <exception cref="ArgumentNullException"><paramref name="function"/> is <see langword="null"/>.</exception>
-    [Experimental("OPENAI001")] // AssistantClient itself is experimental with this ID
     public static FunctionToolDefinition AsOpenAIAssistantsFunctionToolDefinition(this AIFunction function) =>
         OpenAIAssistantChatClient.ToOpenAIAssistantsFunctionToolDefinition(Throw.IfNull(function));
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeConversationClient.cs

@@ -1,7 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using OpenAI.RealtimeConversation;
+using OpenAI.Realtime;
 
 namespace Microsoft.Extensions.AI;
 

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs

@@ -117,6 +117,13 @@ public async Task<ChatResponse> GetResponseAsync(
                         ((List<AIContent>)message.Contents).AddRange(ToAIContents(messageItem.Content));
                         break;
 
+                    case ReasoningResponseItem reasoningItem when reasoningItem.GetSummaryText() is string summary && !string.IsNullOrWhiteSpace(summary):
+                        message.Contents.Add(new TextReasoningContent(summary)
+                        {
+                            RawRepresentation = reasoningItem
+                        });
+                        break;
+
                     case FunctionCallResponseItem functionCall:
                         response.FinishReason ??= ChatFinishReason.ToolCalls;
                         var fcc = FunctionCallContent.CreateFromParsedArguments(
@@ -139,7 +146,7 @@ public async Task<ChatResponse> GetResponseAsync(
 
             if (openAIResponse.Error is { } error)
             {
-                message.Contents.Add(new ErrorContent(error.Message) { ErrorCode = error.Code });
+                message.Contents.Add(new ErrorContent(error.Message) { ErrorCode = error.Code.ToString() });
             }
         }
 
@@ -367,10 +374,11 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt
 
         // Handle strongly-typed properties.
         result.MaxOutputTokenCount ??= options.MaxOutputTokens;
+        result.ParallelToolCallsEnabled ??= options.AllowMultipleToolCalls;
         result.PreviousResponseId ??= options.ConversationId;
-        result.TopP ??= options.TopP;
         result.Temperature ??= options.Temperature;
-        result.ParallelToolCallsEnabled ??= options.AllowMultipleToolCalls;
+        result.TopP ??= options.TopP;
+
         if (options.Instructions is { } instructions)
         {
             result.Instructions = string.IsNullOrEmpty(result.Instructions) ?
@@ -386,22 +394,21 @@ private ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptions? opt
                 switch (tool)
                 {
                     case AIFunction aiFunction:
-                        ResponseTool rtool = ToResponseTool(aiFunction, options);
-                        result.Tools.Add(rtool);
+                        result.Tools.Add(ToResponseTool(aiFunction, options));
                         break;
 
                     case HostedWebSearchTool:
-                        WebSearchToolLocation? location = null;
-                        if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolLocation), out object? objLocation))
+                        WebSearchUserLocation? location = null;
+                        if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchUserLocation), out object? objLocation))
                         {
-                            location = objLocation as WebSearchToolLocation;
+                            location = objLocation as WebSearchUserLocation;
                         }
 
-                        WebSearchToolContextSize? size = null;
-                        if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchToolContextSize), out object? objSize) &&
-                            objSize is WebSearchToolContextSize)
+                        WebSearchContextSize? size = null;
+                        if (tool.AdditionalProperties.TryGetValue(nameof(WebSearchContextSize), out object? objSize) &&
+                            objSize is WebSearchContextSize)
                         {
-                            size = (WebSearchToolContextSize)objSize;
+                            size = (WebSearchContextSize)objSize;
                         }
 
                         result.Tools.Add(ResponseTool.CreateWebSearchTool(location, size));
@@ -522,6 +529,10 @@ private static IEnumerable<ResponseItem> ToOpenAIResponseItems(
                             yield return ResponseItem.CreateAssistantMessageItem(textContent.Text);
                             break;
 
+                        case TextReasoningContent reasoningContent:
+                            yield return ResponseItem.CreateReasoningItem(reasoningContent.Text);
+                            break;
+
                         case FunctionCallContent callContent:
                             yield return ResponseItem.CreateFunctionCallItem(
                                 callContent.CallId,
@@ -555,12 +566,16 @@ private static IEnumerable<ResponseItem> ToOpenAIResponseItems(
                 TotalTokenCount = usage.TotalTokenCount,
             };
 
-            if (usage.OutputTokenDetails is { } outputDetails)
+            if (usage.InputTokenDetails is { } inputDetails)
             {
                 ud.AdditionalCounts ??= [];
+                ud.AdditionalCounts.Add($"{nameof(usage.InputTokenDetails)}.{nameof(inputDetails.CachedTokenCount)}", inputDetails.CachedTokenCount);
+            }
 
-                const string OutputDetails = nameof(usage.OutputTokenDetails);
-                ud.AdditionalCounts.Add($"{OutputDetails}.{nameof(outputDetails.ReasoningTokenCount)}", outputDetails.ReasoningTokenCount);
+            if (usage.OutputTokenDetails is { } outputDetails)
+            {
+                ud.AdditionalCounts ??= [];
+                ud.AdditionalCounts.Add($"{nameof(usage.OutputTokenDetails)}.{nameof(outputDetails.ReasoningTokenCount)}", outputDetails.ReasoningTokenCount);
             }
         }
 
@@ -624,8 +639,7 @@ private static List<ResponseContentPart> ToOpenAIResponsesContent(IList<AIConten
                     break;
 
                 case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase):
-                    parts.Add(ResponseContentPart.CreateInputFilePart(null, $"{Guid.NewGuid():N}.pdf",
-                        BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(dataContent.Uri, OpenAIJsonContext.Default.String))));
+                    parts.Add(ResponseContentPart.CreateInputFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"));
                     break;
 
                 case ErrorContent errorContent when errorContent.ErrorCode == nameof(ResponseContentPartKind.Refusal):