Prepare 9.4.x update for Microsoft.Extensions.AI

eng/MSBuild/LegacySupport.props

@@ -7,6 +7,10 @@
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\LegacySupport\RequiredMemberAttribute\*.cs" LinkBase="LegacySupport\RequiredMemberAttribute" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(InjectSystemIndexOnLegacy)' == 'true' AND ('$(TargetFramework)' == 'net462' or '$(TargetFramework)' == 'netstandard2.0' or '$(TargetFramework)' == 'netcoreapp3.1')">
+    <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\LegacySupport\SystemIndex\*.cs" LinkBase="LegacySupport\SystemIndex" />
+  </ItemGroup>
+
   <ItemGroup Condition="'$(InjectDiagnosticAttributesOnLegacy)' == 'true' AND ('$(TargetFramework)' == 'net462' or '$(TargetFramework)' == 'netstandard2.0' or '$(TargetFramework)' == 'netcoreapp3.1')">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\LegacySupport\DiagnosticAttributes\*.cs" LinkBase="LegacySupport\DiagnosticAttributes" />
   </ItemGroup>

eng/build.proj

@@ -1,6 +1,7 @@
 <Project Sdk="Microsoft.Build.Traversal">
   <ItemGroup>
     <_SnapshotsToExclude Include="$(MSBuildThisFileDirectory)..\test\**\Snapshots\**\*.*proj" />
+    <_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\TemplateSandbox\**\*.*proj" />
 
     <!-- We recursively add all of the projects inside the src directory, except for the exclusions above -->
     <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\**\*.csproj" />
@@ -11,6 +12,6 @@
     <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\Packages\Microsoft.Internal.Extensions.DotNetApiDocs.Transport\Microsoft.Internal.Extensions.DotNetApiDocs.Transport.proj" />
 
     <!-- Add all the projects we want to build as project references, so the traversal SDK can build them -->
-    <ProjectReference Include="@(_ProjectsToBuild)" Exclude="@(_ProjectsToExclude);@(_SnapshotsToExclude)" />
+    <ProjectReference Include="@(_ProjectsToBuild)" Exclude="@(_ProjectsToExclude);@(_SnapshotsToExclude);@(_GeneratedContentToExclude)" />
   </ItemGroup>
-</Project>
+</Project>

eng/pipelines/templates/BuildAndTest.yml

@@ -56,7 +56,25 @@ steps:
               --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config
               --output ${{ parameters.repoTestResultsPath }}/$(Agent.JobName)_CodeCoverageResults/$(Agent.JobName)_cobertura.xml
               "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)"
-      displayName: Run tests
+      displayName: Run unit tests
+
+  - script: ${{ parameters.buildScript }}
+            -pack
+            -configuration ${{ parameters.buildConfig }}
+            -warnAsError 1
+            /bl:${{ parameters.repoLogPath }}/pack.binlog
+            /p:Restore=false /p:Build=false
+            $(_OfficialBuildIdArgs)
+    displayName: Pack
+
+  - ${{ if ne(parameters.skipTests, 'true') }}:
+    - script: ${{ parameters.buildScript }}
+              -integrationTest
+              -configuration ${{ parameters.buildConfig }}
+              -warnAsError 1
+              /bl:${{ parameters.repoLogPath }}/integration_tests.binlog
+              $(_OfficialBuildIdArgs)
+      displayName: Run integration tests
 
     - pwsh: |
         $SourcesDirectory = '$(Build.SourcesDirectory)';
@@ -151,12 +169,11 @@ steps:
       displayName: Build Azure DevOps plugin
 
     - script: ${{ parameters.buildScript }}
-              -pack
               -sign $(_SignArgs)
               -publish $(_PublishArgs)
               -configuration ${{ parameters.buildConfig }}
               -warnAsError 1
-              /bl:${{ parameters.repoLogPath }}/pack.binlog
+              /bl:${{ parameters.repoLogPath }}/publish.binlog
               /p:Restore=false /p:Build=false
               $(_OfficialBuildIdArgs)
-      displayName: Pack, sign, and publish
+      displayName: Sign and publish

src/LegacySupport/SystemIndex/Index.cs

@@ -0,0 +1,160 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+
+#pragma warning disable CS0436 // Type conflicts with imported type
+#pragma warning disable S3427 // Method overloads with default parameter values should not overlap 
+#pragma warning disable SA1642 // Constructor summary documentation should begin with standard text
+#pragma warning disable IDE0011 // Add braces
+#pragma warning disable SA1623 // Property summary documentation should match accessors
+#pragma warning disable IDE0023 // Use block body for conversion operator
+#pragma warning disable S3928 // Parameter names used into ArgumentException constructors should match an existing one
+#pragma warning disable LA0001 // Use the 'Microsoft.Shared.Diagnostics.Throws' class instead of explicitly throwing exception for improved performance
+#pragma warning disable CA1305 // Specify IFormatProvider
+
+namespace System
+{
+    internal readonly struct Index : IEquatable<Index>
+    {
+        private readonly int _value;
+
+        /// <summary>Construct an Index using a value and indicating if the index is from the start or from the end.</summary>
+        /// <param name="value">The index value. it has to be zero or positive number.</param>
+        /// <param name="fromEnd">Indicating if the index is from the start or from the end.</param>
+        /// <remarks>
+        /// If the Index constructed from the end, index value 1 means pointing at the last element and index value 0 means pointing at beyond last element.
+        /// </remarks>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Index(int value, bool fromEnd = false)
+        {
+            if (value < 0)
+            {
+                ThrowValueArgumentOutOfRange_NeedNonNegNumException();
+            }
+
+            if (fromEnd)
+                _value = ~value;
+            else
+                _value = value;
+        }
+
+        // The following private constructors mainly created for perf reason to avoid the checks
+        private Index(int value)
+        {
+            _value = value;
+        }
+
+        /// <summary>Create an Index pointing at first element.</summary>
+        public static Index Start => new Index(0);
+
+        /// <summary>Create an Index pointing at beyond last element.</summary>
+        public static Index End => new Index(~0);
+
+        /// <summary>Create an Index from the start at the position indicated by the value.</summary>
+        /// <param name="value">The index value from the start.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Index FromStart(int value)
+        {
+            if (value < 0)
+            {
+                ThrowValueArgumentOutOfRange_NeedNonNegNumException();
+            }
+
+            return new Index(value);
+        }
+
+        /// <summary>Create an Index from the end at the position indicated by the value.</summary>
+        /// <param name="value">The index value from the end.</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static Index FromEnd(int value)
+        {
+            if (value < 0)
+            {
+                ThrowValueArgumentOutOfRange_NeedNonNegNumException();
+            }
+
+            return new Index(~value);
+        }
+
+        /// <summary>Returns the index value.</summary>
+        public int Value
+        {
+            get
+            {
+                if (_value < 0)
+                    return ~_value;
+                else
+                    return _value;
+            }
+        }
+
+        /// <summary>Indicates whether the index is from the start or the end.</summary>
+        public bool IsFromEnd => _value < 0;
+
+        /// <summary>Calculate the offset from the start using the giving collection length.</summary>
+        /// <param name="length">The length of the collection that the Index will be used with. length has to be a positive value.</param>
+        /// <remarks>
+        /// For performance reason, we don't validate the input length parameter and the returned offset value against negative values.
+        /// we don't validate either the returned offset is greater than the input length.
+        /// It is expected Index will be used with collections which always have non negative length/count. If the returned offset is negative and
+        /// then used to index a collection will get out of range exception which will be same affect as the validation.
+        /// </remarks>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetOffset(int length)
+        {
+            int offset = _value;
+            if (IsFromEnd)
+            {
+                // offset = length - (~value)
+                // offset = length + (~(~value) + 1)
+                // offset = length + value + 1
+
+                offset += length + 1;
+            }
+
+            return offset;
+        }
+
+        /// <summary>Indicates whether the current Index object is equal to another object of the same type.</summary>
+        /// <param name="value">An object to compare with this object.</param>
+        public override bool Equals([NotNullWhen(true)] object? value) => value is Index && _value == ((Index)value)._value;
+
+        /// <summary>Indicates whether the current Index object is equal to another Index object.</summary>
+        /// <param name="other">An object to compare with this object.</param>
+        public bool Equals(Index other) => _value == other._value;
+
+        /// <summary>Returns the hash code for this instance.</summary>
+        public override int GetHashCode() => _value;
+
+        /// <summary>Converts integer number to an Index.</summary>
+        public static implicit operator Index(int value) => FromStart(value);
+
+        /// <summary>Converts the value of the current Index object to its equivalent string representation.</summary>
+        public override string ToString()
+        {
+            if (IsFromEnd)
+                return ToStringFromEnd();
+
+            return ((uint)Value).ToString();
+        }
+
+        private static void ThrowValueArgumentOutOfRange_NeedNonNegNumException()
+        {
+            throw new ArgumentOutOfRangeException("value", "value must be non-negative");
+        }
+
+        private string ToStringFromEnd()
+        {
+#if (!NETSTANDARD2_0 && !NETFRAMEWORK)
+            Span<char> span = stackalloc char[11]; // 1 for ^ and 10 for longest possible uint value
+            bool formatted = ((uint)Value).TryFormat(span.Slice(1), out int charsWritten);
+            span[0] = '^';
+            return new string(span.Slice(0, charsWritten + 1));
+#else
+            return '^' + Value.ToString();
+#endif
+        }
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Release History
 
+## 9.4.3-preview.1.25230.7
+
+- Renamed `ChatThreadId` to `ConversationId` on `ChatResponse`, `ChatResponseUpdate`, and `ChatOptions`.
+- Renamed `EmbeddingGeneratorExtensions` method `GenerateEmbeddingAsync` to `GenerateAsync` and `GenerateEmbeddingVectorAsync` to `GenerateVectorAsync`.
+- Made `AIContent`'s constructor `public` instead of `protected`.
+- Fixed `AIJsonUtilities.CreateJsonSchema` to tolerate `JsonSerializerOptions` instances that don't have a `TypeInfoResolver` already configured.
+
 ## 9.4.0-preview.1.25207.5
 
 - Added `ErrorContent` and `TextReasoningContent`.
@@ -83,4 +90,4 @@
 
 ## 9.0.0-preview.9.24507.7
 
-Initial Preview
+- Initial Preview

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs

@@ -1,12 +1,14 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Represents the options for a chat request.</summary>
+/// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#provide-options">Provide options.</related>
 public class ChatOptions
 {
     /// <summary>Gets or sets an optional identifier used to associate a request with an existing conversation.</summary>
@@ -20,6 +22,7 @@ public string? ChatThreadId
     }
 
     /// <summary>Gets or sets an optional identifier used to associate a request with an existing conversation.</summary>
+    /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#stateless-vs-stateful-clients">Stateless vs. stateful clients.</related>
     public string? ConversationId { get; set; }
 
     /// <summary>Gets or sets the temperature for generating chat responses.</summary>
@@ -115,9 +118,30 @@ public string? ChatThreadId
     public ChatToolMode? ToolMode { get; set; }
 
     /// <summary>Gets or sets the list of tools to include with a chat request.</summary>
+    /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#tool-calling">Tool calling.</related>
     [JsonIgnore]
     public IList<AITool>? Tools { get; set; }
 
+    /// <summary>
+    /// Gets or sets a callback responsible of creating the raw representation of the chat options from an underlying implementation.
+    /// </summary>
+    /// <remarks>
+    /// The underlying <see cref="IChatClient" /> implementation may have its own representation of options.
+    /// When <see cref="IChatClient.GetResponseAsync" /> or <see cref="IChatClient.GetStreamingResponseAsync" />
+    /// is invoked with a <see cref="ChatOptions" />, that implementation may convert the provided options into
+    /// its own representation in order to use it while performing the operation. For situations where a consumer knows
+    /// which concrete <see cref="IChatClient" /> is being used and how it represents options, a new instance of that
+    /// implementation-specific options type may be returned by this callback, for the <see cref="IChatClient" />
+    /// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options
+    /// instance further based on other settings supplied on this <see cref="ChatOptions" /> instance or from other inputs,
+    /// like the enumerable of <see cref="ChatMessage"/>s, therefore, its **strongly recommended** to not return shared instances
+    /// and instead make the callback return a new instance per each call.
+    /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed
+    /// properties on <see cref="ChatOptions" />.
+    /// </remarks>
+    [JsonIgnore]
+    public Func<IChatClient, object?>? RawRepresentationFactory { get; set; }
+
     /// <summary>Gets or sets any additional properties associated with the options.</summary>
     public AdditionalPropertiesDictionary? AdditionalProperties { get; set; }
 
@@ -144,6 +168,7 @@ public virtual ChatOptions Clone()
             ModelId = ModelId,
             AllowMultipleToolCalls = AllowMultipleToolCalls,
             ToolMode = ToolMode,
+            RawRepresentationFactory = RawRepresentationFactory,
             AdditionalProperties = AdditionalProperties?.Clone(),
         };
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs

@@ -92,6 +92,7 @@ public string? ChatThreadId
     /// or may not differ on every response, depending on whether the underlying provider uses a fixed ID for each conversation
     /// or updates it for each message.
     /// </remarks>
+    /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#stateless-vs-stateful-clients">Stateless vs. stateful clients.</related>
     public string? ConversationId { get; set; }
 
     /// <summary>Gets or sets the model ID used in the creation of the chat response.</summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/DelegatingChatClient.cs

@@ -16,6 +16,7 @@ namespace Microsoft.Extensions.AI;
 /// This is recommended as a base type when building clients that can be chained around an underlying <see cref="IChatClient"/>.
 /// The default implementation simply passes each call to the inner client instance.
 /// </remarks>
+/// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#custom-ichatclient-middleware">Custom IChatClient middleware.</related>
 public class DelegatingChatClient : IChatClient
 {
     /// <summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/IChatClient.cs

@@ -24,6 +24,7 @@ namespace Microsoft.Extensions.AI;
 /// </para>
 /// </remarks>
 /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/quickstarts/build-chat-app">Build an AI chat app with .NET.</related>
+/// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#the-ichatclient-interface">The IChatClient interface.</related>
 public interface IChatClient : IDisposable
 {
     /// <summary>Sends chat messages and returns the response.</summary>
@@ -32,6 +33,7 @@ public interface IChatClient : IDisposable
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>The response messages generated by the client.</returns>
     /// <exception cref="ArgumentNullException"><paramref name="messages"/> is <see langword="null"/>.</exception>
+    /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#request-a-chat-response">Request a chat response.</related>
     Task<ChatResponse> GetResponseAsync(
         IEnumerable<ChatMessage> messages,
         ChatOptions? options = null,
@@ -43,6 +45,7 @@ Task<ChatResponse> GetResponseAsync(
     /// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
     /// <returns>The response messages generated by the client.</returns>
     /// <exception cref="ArgumentNullException"><paramref name="messages"/> is <see langword="null"/>.</exception>
+    /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/microsoft-extensions-ai#request-a-streaming-chat-response">Request a streaming chat response.</related>
     IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
         IEnumerable<ChatMessage> messages,
         ChatOptions? options = null,