diff --git a/eng/MSBuild/LegacySupport.props b/eng/MSBuild/LegacySupport.props index 2983903a196..8c05622c41c 100644 --- a/eng/MSBuild/LegacySupport.props +++ b/eng/MSBuild/LegacySupport.props @@ -7,6 +7,10 @@ + + + + diff --git a/eng/build.proj b/eng/build.proj index 2f95df76500..e59868907db 100644 --- a/eng/build.proj +++ b/eng/build.proj @@ -1,6 +1,7 @@ <_SnapshotsToExclude Include="$(MSBuildThisFileDirectory)..\test\**\Snapshots\**\*.*proj" /> + <_GeneratedContentToExclude Include="$(MSBuildThisFileDirectory)..\test\**\TemplateSandbox\**\*.*proj" /> <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\**\*.csproj" /> @@ -11,6 +12,6 @@ <_ProjectsToBuild Include="$(MSBuildThisFileDirectory)..\src\Packages\Microsoft.Internal.Extensions.DotNetApiDocs.Transport\Microsoft.Internal.Extensions.DotNetApiDocs.Transport.proj" /> - + - \ No newline at end of file + diff --git a/eng/pipelines/templates/BuildAndTest.yml b/eng/pipelines/templates/BuildAndTest.yml index 92717160bf4..bd008ead075 100644 --- a/eng/pipelines/templates/BuildAndTest.yml +++ b/eng/pipelines/templates/BuildAndTest.yml @@ -56,7 +56,25 @@ steps: --settings $(Build.SourcesDirectory)/eng/CodeCoverage.config --output ${{ parameters.repoTestResultsPath }}/$(Agent.JobName)_CodeCoverageResults/$(Agent.JobName)_cobertura.xml "${{ parameters.buildScript }} -test -configuration ${{ parameters.buildConfig }} /bl:${{ parameters.repoLogPath }}/tests.binlog $(_OfficialBuildIdArgs)" - displayName: Run tests + displayName: Run unit tests + + - script: ${{ parameters.buildScript }} + -pack + -configuration ${{ parameters.buildConfig }} + -warnAsError 1 + /bl:${{ parameters.repoLogPath }}/pack.binlog + /p:Restore=false /p:Build=false + $(_OfficialBuildIdArgs) + displayName: Pack + + - ${{ if ne(parameters.skipTests, 'true') }}: + - script: ${{ parameters.buildScript }} + -integrationTest + -configuration ${{ parameters.buildConfig }} + -warnAsError 1 + /bl:${{ parameters.repoLogPath }}/integration_tests.binlog + $(_OfficialBuildIdArgs) + displayName: Run integration tests - pwsh: | $SourcesDirectory = '$(Build.SourcesDirectory)'; @@ -151,12 +169,11 @@ steps: displayName: Build Azure DevOps plugin - script: ${{ parameters.buildScript }} - -pack -sign $(_SignArgs) -publish $(_PublishArgs) -configuration ${{ parameters.buildConfig }} -warnAsError 1 - /bl:${{ parameters.repoLogPath }}/pack.binlog + /bl:${{ parameters.repoLogPath }}/publish.binlog /p:Restore=false /p:Build=false $(_OfficialBuildIdArgs) - displayName: Pack, sign, and publish + displayName: Sign and publish diff --git a/src/LegacySupport/SystemIndex/Index.cs b/src/LegacySupport/SystemIndex/Index.cs new file mode 100644 index 00000000000..7285d669e71 --- /dev/null +++ b/src/LegacySupport/SystemIndex/Index.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +#pragma warning disable CS0436 // Type conflicts with imported type +#pragma warning disable S3427 // Method overloads with default parameter values should not overlap +#pragma warning disable SA1642 // Constructor summary documentation should begin with standard text +#pragma warning disable IDE0011 // Add braces +#pragma warning disable SA1623 // Property summary documentation should match accessors +#pragma warning disable IDE0023 // Use block body for conversion operator +#pragma warning disable S3928 // Parameter names used into ArgumentException constructors should match an existing one +#pragma warning disable LA0001 // Use the 'Microsoft.Shared.Diagnostics.Throws' class instead of explicitly throwing exception for improved performance +#pragma warning disable CA1305 // Specify IFormatProvider + +namespace System +{ + internal readonly struct Index : IEquatable + { + private readonly int _value; + + /// Construct an Index using a value and indicating if the index is from the start or from the end. + /// The index value. it has to be zero or positive number. + /// Indicating if the index is from the start or from the end. + /// + /// If the Index constructed from the end, index value 1 means pointing at the last element and index value 0 means pointing at beyond last element. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Index(int value, bool fromEnd = false) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + if (fromEnd) + _value = ~value; + else + _value = value; + } + + // The following private constructors mainly created for perf reason to avoid the checks + private Index(int value) + { + _value = value; + } + + /// Create an Index pointing at first element. + public static Index Start => new Index(0); + + /// Create an Index pointing at beyond last element. + public static Index End => new Index(~0); + + /// Create an Index from the start at the position indicated by the value. + /// The index value from the start. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromStart(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(value); + } + + /// Create an Index from the end at the position indicated by the value. + /// The index value from the end. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromEnd(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(~value); + } + + /// Returns the index value. + public int Value + { + get + { + if (_value < 0) + return ~_value; + else + return _value; + } + } + + /// Indicates whether the index is from the start or the end. + public bool IsFromEnd => _value < 0; + + /// Calculate the offset from the start using the giving collection length. + /// The length of the collection that the Index will be used with. length has to be a positive value. + /// + /// For performance reason, we don't validate the input length parameter and the returned offset value against negative values. + /// we don't validate either the returned offset is greater than the input length. + /// It is expected Index will be used with collections which always have non negative length/count. If the returned offset is negative and + /// then used to index a collection will get out of range exception which will be same affect as the validation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetOffset(int length) + { + int offset = _value; + if (IsFromEnd) + { + // offset = length - (~value) + // offset = length + (~(~value) + 1) + // offset = length + value + 1 + + offset += length + 1; + } + + return offset; + } + + /// Indicates whether the current Index object is equal to another object of the same type. + /// An object to compare with this object. + public override bool Equals([NotNullWhen(true)] object? value) => value is Index && _value == ((Index)value)._value; + + /// Indicates whether the current Index object is equal to another Index object. + /// An object to compare with this object. + public bool Equals(Index other) => _value == other._value; + + /// Returns the hash code for this instance. + public override int GetHashCode() => _value; + + /// Converts integer number to an Index. + public static implicit operator Index(int value) => FromStart(value); + + /// Converts the value of the current Index object to its equivalent string representation. + public override string ToString() + { + if (IsFromEnd) + return ToStringFromEnd(); + + return ((uint)Value).ToString(); + } + + private static void ThrowValueArgumentOutOfRange_NeedNonNegNumException() + { + throw new ArgumentOutOfRangeException("value", "value must be non-negative"); + } + + private string ToStringFromEnd() + { +#if (!NETSTANDARD2_0 && !NETFRAMEWORK) + Span span = stackalloc char[11]; // 1 for ^ and 10 for longest possible uint value + bool formatted = ((uint)Value).TryFormat(span.Slice(1), out int charsWritten); + span[0] = '^'; + return new string(span.Slice(0, charsWritten + 1)); +#else + return '^' + Value.ToString(); +#endif + } + } +} \ No newline at end of file diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md index 7df054549c0..c6639273d70 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md @@ -1,5 +1,12 @@ # Release History +## 9.4.3-preview.1.25230.7 + +- Renamed `ChatThreadId` to `ConversationId` on `ChatResponse`, `ChatResponseUpdate`, and `ChatOptions`. +- Renamed `EmbeddingGeneratorExtensions` method `GenerateEmbeddingAsync` to `GenerateAsync` and `GenerateEmbeddingVectorAsync` to `GenerateVectorAsync`. +- Made `AIContent`'s constructor `public` instead of `protected`. +- Fixed `AIJsonUtilities.CreateJsonSchema` to tolerate `JsonSerializerOptions` instances that don't have a `TypeInfoResolver` already configured. + ## 9.4.0-preview.1.25207.5 - Added `ErrorContent` and `TextReasoningContent`. @@ -83,4 +90,4 @@ ## 9.0.0-preview.9.24507.7 -Initial Preview +- Initial Preview diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs index 8b3bef838dd..f2eeffe9dbf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs @@ -1,12 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; /// Represents the options for a chat request. +/// Provide options. public class ChatOptions { /// Gets or sets an optional identifier used to associate a request with an existing conversation. @@ -20,6 +22,7 @@ public string? ChatThreadId } /// Gets or sets an optional identifier used to associate a request with an existing conversation. + /// Stateless vs. stateful clients. public string? ConversationId { get; set; } /// Gets or sets the temperature for generating chat responses. @@ -115,9 +118,30 @@ public string? ChatThreadId public ChatToolMode? ToolMode { get; set; } /// Gets or sets the list of tools to include with a chat request. + /// Tool calling. [JsonIgnore] public IList? Tools { get; set; } + /// + /// Gets or sets a callback responsible of creating the raw representation of the chat options from an underlying implementation. + /// + /// + /// The underlying implementation may have its own representation of options. + /// When or + /// is invoked with a , that implementation may convert the provided options into + /// its own representation in order to use it while performing the operation. For situations where a consumer knows + /// which concrete is being used and how it represents options, a new instance of that + /// implementation-specific options type may be returned by this callback, for the + /// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options + /// instance further based on other settings supplied on this instance or from other inputs, + /// like the enumerable of s, therefore, its **strongly recommended** to not return shared instances + /// and instead make the callback return a new instance per each call. + /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed + /// properties on . + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } + /// Gets or sets any additional properties associated with the options. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } @@ -144,6 +168,7 @@ public virtual ChatOptions Clone() ModelId = ModelId, AllowMultipleToolCalls = AllowMultipleToolCalls, ToolMode = ToolMode, + RawRepresentationFactory = RawRepresentationFactory, AdditionalProperties = AdditionalProperties?.Clone(), }; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs index 92be7a5145c..5e0e80beac9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponse.cs @@ -92,6 +92,7 @@ public string? ChatThreadId /// or may not differ on every response, depending on whether the underlying provider uses a fixed ID for each conversation /// or updates it for each message. /// + /// Stateless vs. stateful clients. public string? ConversationId { get; set; } /// Gets or sets the model ID used in the creation of the chat response. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/DelegatingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/DelegatingChatClient.cs index 23768dd8da7..112e846d41f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/DelegatingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/DelegatingChatClient.cs @@ -16,6 +16,7 @@ namespace Microsoft.Extensions.AI; /// This is recommended as a base type when building clients that can be chained around an underlying . /// The default implementation simply passes each call to the inner client instance. /// +/// Custom IChatClient middleware. public class DelegatingChatClient : IChatClient { /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/IChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/IChatClient.cs index bb0fe6a428c..b4354e22a43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/IChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/IChatClient.cs @@ -24,6 +24,7 @@ namespace Microsoft.Extensions.AI; /// /// /// Build an AI chat app with .NET. +/// The IChatClient interface. public interface IChatClient : IDisposable { /// Sends chat messages and returns the response. @@ -32,6 +33,7 @@ public interface IChatClient : IDisposable /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. /// is . + /// Request a chat response. Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, @@ -43,6 +45,7 @@ Task GetResponseAsync( /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. /// is . + /// Request a streaming chat response. IAsyncEnumerable GetStreamingResponseAsync( IEnumerable messages, ChatOptions? options = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs index 6353586208f..5bbde1e1444 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs @@ -2,14 +2,22 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +#if NET +using System.Buffers; +using System.Buffers.Text; +#endif using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +#if !NET +using System.Runtime.InteropServices; +#endif using System.Text.Json.Serialization; using Microsoft.Shared.Diagnostics; #pragma warning disable S3996 // URI properties should not be strings #pragma warning disable CA1054 // URI-like parameters should not be strings #pragma warning disable CA1056 // URI-like properties should not be strings +#pragma warning disable CA1307 // Specify StringComparison for clarity namespace Microsoft.Extensions.AI; @@ -70,39 +78,35 @@ public DataContent(Uri uri, string? mediaType = null) [JsonConstructor] public DataContent([StringSyntax(StringSyntaxAttribute.Uri)] string uri, string? mediaType = null) { + // Store and validate the data URI. _uri = Throw.IfNullOrWhitespace(uri); - if (!uri.StartsWith(DataUriParser.Scheme, StringComparison.OrdinalIgnoreCase)) { Throw.ArgumentException(nameof(uri), "The provided URI is not a data URI."); } + // Parse the data URI to extract the data and media type. _dataUri = DataUriParser.Parse(uri.AsMemory()); + // Validate and store the media type. + mediaType ??= _dataUri.MediaType; if (mediaType is null) { - mediaType = _dataUri.MediaType; - if (mediaType is null) - { - Throw.ArgumentNullException(nameof(mediaType), $"{nameof(uri)} did not contain a media type, and {nameof(mediaType)} was not provided."); - } - } - else - { - if (mediaType != _dataUri.MediaType) - { - // If the data URI contains a media type that's different from a non-null media type - // explicitly provided, prefer the one explicitly provided as an override. - - // Extract the bytes from the data URI and null out the uri. - // Then we'll lazily recreate it later if needed based on the updated media type. - _data = _dataUri.ToByteArray(); - _dataUri = null; - _uri = null; - } + Throw.ArgumentNullException(nameof(mediaType), $"{nameof(uri)} did not contain a media type, and {nameof(mediaType)} was not provided."); } MediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType); + + if (!_dataUri.IsBase64 || mediaType != _dataUri.MediaType) + { + // In rare cases, the data URI may contain non-base64 data, in which case we + // want to normalize it to base64. The supplied media type may also be different + // from the one in the data URI. In either case, we extract the bytes from the data URI + // and then throw away the uri; we'll recreate it lazily in the canonical form. + _data = _dataUri.ToByteArray(); + _dataUri = null; + _uri = null; + } } /// @@ -134,9 +138,8 @@ public DataContent(ReadOnlyMemory data, string mediaType) /// Gets the data URI for this . /// - /// The returned URI is always a valid URI string, even if the instance was constructed from a - /// or from a . In the case of a , this property returns a data URI containing - /// that data. + /// The returned URI is always a valid data URI string, even if the instance was constructed from a + /// or from a . /// [StringSyntax(StringSyntaxAttribute.Uri)] public string Uri @@ -145,27 +148,26 @@ public string Uri { if (_uri is null) { - if (_dataUri is null) - { - Debug.Assert(_data is not null, "Expected _data to be initialized."); - _uri = string.Concat("data:", MediaType, ";base64,", Convert.ToBase64String(_data.GetValueOrDefault() -#if NET - .Span)); -#else - .Span.ToArray())); -#endif - } - else - { - _uri = _dataUri.IsBase64 ? + Debug.Assert(_data is not null, "Expected _data to be initialized."); + ReadOnlyMemory data = _data.GetValueOrDefault(); + #if NET - $"data:{MediaType};base64,{_dataUri.Data.Span}" : - $"data:{MediaType};,{_dataUri.Data.Span}"; + char[] array = ArrayPool.Shared.Rent( + "data:".Length + MediaType.Length + ";base64,".Length + Base64.GetMaxEncodedToUtf8Length(data.Length)); + + bool wrote = array.AsSpan().TryWrite($"data:{MediaType};base64,", out int prefixLength); + wrote |= Convert.TryToBase64Chars(data.Span, array.AsSpan(prefixLength), out int dataLength); + Debug.Assert(wrote, "Expected to successfully write the data URI."); + _uri = array.AsSpan(0, prefixLength + dataLength).ToString(); + + ArrayPool.Shared.Return(array); #else - $"data:{MediaType};base64,{_dataUri.Data}" : - $"data:{MediaType};,{_dataUri.Data}"; + string base64 = MemoryMarshal.TryGetArray(data, out ArraySegment segment) ? + Convert.ToBase64String(segment.Array!, segment.Offset, segment.Count) : + Convert.ToBase64String(data.ToArray()); + + _uri = $"data:{MediaType};base64,{base64}"; #endif - } } return _uri; @@ -205,6 +207,20 @@ public ReadOnlyMemory Data } } + /// Gets the data represented by this instance as a Base64 character sequence. + /// The base64 representation of the data. + [JsonIgnore] + public ReadOnlyMemory Base64Data + { + get + { + string uri = Uri; + int pos = uri.IndexOf(','); + Debug.Assert(pos >= 0, "Expected comma to be present in the URI."); + return uri.AsMemory(pos + 1); + } + } + /// Gets a string representing this instance to display in the debugger. [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs new file mode 100644 index 00000000000..2261fd97949 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Collections; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents an embedding composed of a bit vector. +public sealed class BinaryEmbedding : Embedding +{ + /// The embedding vector this embedding represents. + private BitArray _vector; + + /// Initializes a new instance of the class with the embedding vector. + /// The embedding vector this embedding represents. + /// is . + public BinaryEmbedding(BitArray vector) + { + _vector = Throw.IfNull(vector); + } + + /// Gets or sets the embedding vector this embedding represents. + [JsonConverter(typeof(VectorConverter))] + public BitArray Vector + { + get => _vector; + set => _vector = Throw.IfNull(value); + } + + /// + [JsonIgnore] + public override int Dimensions => _vector.Length; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class VectorConverter : JsonConverter + { + /// + public override BitArray Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + _ = Throw.IfNull(typeToConvert); + _ = Throw.IfNull(options); + + if (reader.TokenType != JsonTokenType.String) + { + throw new JsonException("Expected string property."); + } + + ReadOnlySpan utf8; + byte[]? tmpArray = null; + if (!reader.HasValueSequence && !reader.ValueIsEscaped) + { + utf8 = reader.ValueSpan; + } + else + { + // This path should be rare. + int length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; + tmpArray = ArrayPool.Shared.Rent(length); + utf8 = tmpArray.AsSpan(0, reader.CopyString(tmpArray)); + } + + BitArray result = new(utf8.Length); + + for (int i = 0; i < utf8.Length; i++) + { + result[i] = utf8[i] switch + { + (byte)'0' => false, + (byte)'1' => true, + _ => throw new JsonException("Expected binary character sequence.") + }; + } + + if (tmpArray is not null) + { + ArrayPool.Shared.Return(tmpArray); + } + + return result; + } + + /// + public override void Write(Utf8JsonWriter writer, BitArray value, JsonSerializerOptions options) + { + _ = Throw.IfNull(writer); + _ = Throw.IfNull(value); + _ = Throw.IfNull(options); + + int length = value.Length; + + byte[] tmpArray = ArrayPool.Shared.Rent(length); + + Span utf8 = tmpArray.AsSpan(0, length); + for (int i = 0; i < utf8.Length; i++) + { + utf8[i] = value[i] ? (byte)'1' : (byte)'0'; + } + + writer.WriteStringValue(utf8); + + ArrayPool.Shared.Return(tmpArray); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs index 19b8feaa182..d6596e1e53e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -9,13 +10,15 @@ namespace Microsoft.Extensions.AI; /// Represents an embedding generated by a . /// This base class provides metadata about the embedding. Derived types provide the concrete data contained in the embedding. [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(BinaryEmbedding), typeDiscriminator: "binary")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "uint8")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "int8")] #if NET -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "halves")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float16")] #endif -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "floats")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "doubles")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "bytes")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "sbytes")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float32")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float64")] +[DebuggerDisplay("Dimensions = {Dimensions}")] public class Embedding { /// Initializes a new instance of the class. @@ -26,6 +29,13 @@ protected Embedding() /// Gets or sets a timestamp at which the embedding was created. public DateTimeOffset? CreatedAt { get; set; } + /// Gets the dimensionality of the embedding vector. + /// + /// This value corresponds to the number of elements in the embedding vector. + /// + [JsonIgnore] + public virtual int Dimensions { get; } + /// Gets or sets the model ID using in the creation of the embedding. public string? ModelId { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs index c80e20dfda4..22bc02f2f3f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -19,4 +20,8 @@ public Embedding(ReadOnlyMemory vector) /// Gets or sets the embedding vector this embedding represents. public ReadOnlyMemory Vector { get; set; } + + /// + [JsonIgnore] + public override int Dimensions => Vector.Length; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/IEmbeddingGenerator{TInput,TEmbedding}.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/IEmbeddingGenerator{TInput,TEmbedding}.cs index ff3910ae737..ad6b9951642 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/IEmbeddingGenerator{TInput,TEmbedding}.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/IEmbeddingGenerator{TInput,TEmbedding}.cs @@ -24,6 +24,7 @@ namespace Microsoft.Extensions.AI; /// no instances are used which might employ such mutation. /// /// +/// The IEmbeddingGenerator interface. public interface IEmbeddingGenerator : IEmbeddingGenerator where TEmbedding : Embedding { @@ -33,6 +34,7 @@ public interface IEmbeddingGenerator : IEmbeddingGenerato /// The to monitor for cancellation requests. The default is . /// The generated embeddings. /// is . + /// Create embeddings. Task> GenerateAsync( IEnumerable values, EmbeddingGenerationOptions? options = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj index 27a2c5d0513..dc6896b7f53 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj @@ -28,6 +28,7 @@ true true true + true diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/README.md b/src/Libraries/Microsoft.Extensions.AI.Abstractions/README.md index 08caff50fb0..94a0c53e162 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/README.md @@ -396,10 +396,10 @@ while (true) } ``` -For stateful services, you may know ahead of time an identifier used for the relevant conversation. That identifier can be put into `ChatOptions.ChatThreadId`. +For stateful services, you may know ahead of time an identifier used for the relevant conversation. That identifier can be put into `ChatOptions.ConversationId`. Usage then follows the same pattern, except there's no need to maintain a history manually. ```csharp -ChatOptions options = new() { ChatThreadId = "my-conversation-id" }; +ChatOptions options = new() { ConversationId = "my-conversation-id" }; while (true) { Console.Write("Q: "); @@ -409,8 +409,8 @@ while (true) } ``` -Some services may support automatically creating a thread ID for a request that doesn't have one. In such cases, you can transfer the `ChatResponse.ChatThreadId` over -to the `ChatOptions.ChatThreadId` for subsequent requests, e.g. +Some services may support automatically creating a thread ID for a request that doesn't have one. In such cases, you can transfer the `ChatResponse.ConversationId` over +to the `ChatOptions.ConversationId` for subsequent requests, e.g. ```csharp ChatOptions options = new(); while (true) @@ -421,13 +421,13 @@ while (true) ChatResponse response = await client.GetResponseAsync(message, options); Console.WriteLine(response); - options.ChatThreadId = response.ChatThreadId; + options.ConversationId = response.ConversationId; } ``` -If you don't know ahead of time whether the service is stateless or stateful, both can be accomodated by checking the response `ChatThreadId` -and acting based on its value. Here, if the response `ChatThreadId` is set, then that value is propagated to the options and the history -cleared so as to not resend the same history again. If, however, the `ChatThreadId` is not set, then the response message is added to the +If you don't know ahead of time whether the service is stateless or stateful, both can be accomodated by checking the response `ConversationId` +and acting based on its value. Here, if the response `ConversationId` is set, then that value is propagated to the options and the history +cleared so as to not resend the same history again. If, however, the `ConversationId` is not set, then the response message is added to the history so that it's sent back to the service on the next turn. ```csharp List history = []; @@ -440,8 +440,8 @@ while (true) ChatResponse response = await client.GetResponseAsync(history); Console.WriteLine(response); - options.ChatThreadId = response.ChatThreadId; - if (response.ChatThreadId is not null) + options.ConversationId = response.ConversationId; + if (response.ConversationId is not null) { history.Clear(); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs new file mode 100644 index 00000000000..a1aaeff26ac --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines a cache for JSON schemas transformed according to the specified policy. +/// +/// +/// +/// This cache stores weak references from AI abstractions that declare JSON schemas such as or +/// to their corresponding JSON schemas transformed according to the specified policy. It is intended for use by +/// implementations that enforce vendor-specific restrictions on what constitutes a valid JSON schema for a given function or response format. +/// +/// +/// It is recommended implementations with schema transformation requirements should create a single static instance of this cache. +/// +/// +public sealed class AIJsonSchemaTransformCache +{ + private readonly ConditionalWeakTable _functionSchemaCache = new(); + private readonly ConditionalWeakTable _responseFormatCache = new(); + + private readonly ConditionalWeakTable.CreateValueCallback _functionSchemaCreateValueCallback; + private readonly ConditionalWeakTable.CreateValueCallback _responseFormatCreateValueCallback; + + /// + /// Initializes a new instance of the class with the specified options. + /// + /// The options governing schema transformation. + public AIJsonSchemaTransformCache(AIJsonSchemaTransformOptions transformOptions) + { + _ = Throw.IfNull(transformOptions); + + if (transformOptions == AIJsonSchemaTransformOptions.Default) + { + Throw.ArgumentException(nameof(transformOptions), "The options instance does not specify any transformations."); + } + + TransformOptions = transformOptions; + _functionSchemaCreateValueCallback = function => AIJsonUtilities.TransformSchema(function.JsonSchema, TransformOptions); + _responseFormatCreateValueCallback = responseFormat => AIJsonUtilities.TransformSchema(responseFormat.Schema!.Value, TransformOptions); + } + + /// + /// Gets the options governing schema transformation. + /// + public AIJsonSchemaTransformOptions TransformOptions { get; } + + /// + /// Gets or creates a transformed JSON schema for the specified instance. + /// + /// The function whose JSON schema we want to transform. + /// The transformed JSON schema corresponding to . + public JsonElement GetOrCreateTransformedSchema(AIFunction function) + { + _ = Throw.IfNull(function); + return (JsonElement)_functionSchemaCache.GetValue(function, _functionSchemaCreateValueCallback); + } + + /// + /// Gets or creates a transformed JSON schema for the specified instance. + /// + /// The response format whose JSON schema we want to transform. + /// The transformed JSON schema corresponding to . + public JsonElement? GetOrCreateTransformedSchema(ChatResponseFormatJson responseFormat) + { + _ = Throw.IfNull(responseFormat); + return responseFormat.Schema is not null + ? (JsonElement?)_responseFormatCache.GetValue(responseFormat, _responseFormatCreateValueCallback) + : null; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs new file mode 100644 index 00000000000..4cfd08e160b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable CA1815 // Override equals and operator equals on value types + +using System; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines the context for transforming a schema node withing a larger schema document. +/// +/// +/// This struct is being passed to the user-provided +/// callback by the method and cannot be instantiated directly. +/// +public readonly struct AIJsonSchemaTransformContext +{ + private readonly string[] _path; + + internal AIJsonSchemaTransformContext(string[] path) + { + _path = path; + } + + /// + /// Gets the path to the schema document currently being generated. + /// + public ReadOnlySpan Path => _path; + + /// + /// Gets the containing property name if the current schema is a property of an object. + /// + public string? PropertyName => Path is [.., "properties", string name] ? name : null; + + /// + /// Gets a value indicating whether the current schema is a collection element. + /// + public bool IsCollectionElementSchema => Path is [.., "items"]; + + /// + /// Gets a value indicating whether the current schema is a dictionary value. + /// + public bool IsDictionaryValueSchema => Path is [.., "additionalProperties"]; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs new file mode 100644 index 00000000000..c7a035cbbed --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable S1067 // Expressions should not be too complex + +using System; +using System.Text.Json.Nodes; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides options for configuring the behavior of JSON schema transformation functionality. +/// +public sealed record class AIJsonSchemaTransformOptions +{ + /// + /// Gets a callback that is invoked for every schema that is generated within the type graph. + /// + public Func? TransformSchemaNode { get; init; } + + /// + /// Gets a value indicating whether to convert boolean schemas to equivalent object-based representations. + /// + public bool ConvertBooleanSchemas { get; init; } + + /// + /// Gets a value indicating whether to generate schemas with the additionalProperties set to false for .NET objects. + /// + public bool DisallowAdditionalProperties { get; init; } + + /// + /// Gets a value indicating whether to mark all properties as required in the schema. + /// + public bool RequireAllProperties { get; init; } + + /// + /// Gets a value indicating whether to substitute nullable "type" keywords with OpenAPI 3.0 style "nullable" keywords in the schema. + /// + public bool UseNullableKeyword { get; init; } + + /// + /// Gets the default options instance. + /// + internal static AIJsonSchemaTransformOptions Default { get; } = new(); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs similarity index 96% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs index f0764d6fae8..fe17a2ad449 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs @@ -34,6 +34,7 @@ public static partial class AIJsonUtilities private const string PatternPropertyName = "pattern"; private const string EnumPropertyName = "enum"; private const string PropertiesPropertyName = "properties"; + private const string ItemsPropertyName = "items"; private const string RequiredPropertyName = "required"; private const string AdditionalPropertiesPropertyName = "additionalProperties"; private const string DefaultPropertyName = "default"; @@ -446,21 +447,23 @@ private static JsonElement ParseJsonElement(ReadOnlySpan utf8Json) return JsonElement.ParseValue(ref reader); } + [UnconditionalSuppressMessage("Trimming", "IL2072:Target parameter argument does not satisfy 'DynamicallyAccessedMembersAttribute' in call to target method.", + Justification = "Called conditionally on structs whose default ctor never gets trimmed.")] private static object? GetDefaultValueNormalized(ParameterInfo parameterInfo) { // Taken from https://github.com/dotnet/runtime/blob/eff415bfd667125c1565680615a6f19152645fbf/src/libraries/System.Text.Json/Common/ReflectionExtensions.cs#L288-L317 Type parameterType = parameterInfo.ParameterType; object? defaultValue = parameterInfo.DefaultValue; - if (defaultValue is null) + if (defaultValue is null || (defaultValue == DBNull.Value && parameterType != typeof(DBNull))) { - return null; - } - - // DBNull.Value is sometimes used as the default value (returned by reflection) of nullable params in place of null. - if (defaultValue == DBNull.Value && parameterType != typeof(DBNull)) - { - return null; + return parameterType.IsValueType +#if NET + ? RuntimeHelpers.GetUninitializedObject(parameterType) +#else + ? System.Runtime.Serialization.FormatterServices.GetUninitializedObject(parameterType) +#endif + : null; } // Default values of enums or nullable enums are represented using the underlying type and need to be cast explicitly diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs new file mode 100644 index 00000000000..5669a3fb264 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs @@ -0,0 +1,188 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +public static partial class AIJsonUtilities +{ + /// + /// Transforms the given JSON schema based on the provided options. + /// + /// The schema document to transform. + /// The options governing schema transformation. + /// A new schema document with transformations applied. + /// The schema and any nested schemas are transformed using depth-first traversal. + public static JsonElement TransformSchema(JsonElement schema, AIJsonSchemaTransformOptions transformOptions) + { + _ = Throw.IfNull(transformOptions); + + if (transformOptions == AIJsonSchemaTransformOptions.Default) + { + Throw.ArgumentException(nameof(transformOptions), "The options instance does not specify any transformations."); + } + + JsonNode? nodeSchema = JsonSerializer.SerializeToNode(schema, JsonContext.Default.JsonElement); + List? path = transformOptions.TransformSchemaNode is not null ? [] : null; + JsonNode transformedSchema = TransformSchemaCore(nodeSchema, transformOptions, path); + return JsonSerializer.Deserialize(transformedSchema, JsonContext.Default.JsonElement); + } + + private static JsonNode TransformSchemaCore(JsonNode? schema, AIJsonSchemaTransformOptions transformOptions, List? path) + { + switch (schema?.GetValueKind()) + { + case JsonValueKind.False: + if (transformOptions.ConvertBooleanSchemas) + { + schema = new JsonObject { [NotPropertyName] = (JsonNode)true }; + } + + break; + + case JsonValueKind.True: + if (transformOptions.ConvertBooleanSchemas) + { + schema = new JsonObject(); + } + + break; + + case JsonValueKind.Object: + JsonObject schemaObj = (JsonObject)schema; + JsonObject? properties = null; + + // Step 1. Recursively apply transformations to any nested schemas we might be able to detect. + if (schemaObj.TryGetPropertyValue(PropertiesPropertyName, out JsonNode? props) && props is JsonObject propsObj) + { + properties = propsObj; + path?.Add(PropertiesPropertyName); + foreach (var prop in properties.ToArray()) + { + path?.Add(prop.Key); + properties[prop.Key] = TransformSchemaCore(prop.Value, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(ItemsPropertyName, out JsonNode? itemsSchema)) + { + path?.Add(ItemsPropertyName); + schemaObj[ItemsPropertyName] = TransformSchemaCore(itemsSchema, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(AdditionalPropertiesPropertyName, out JsonNode? additionalProps) && + additionalProps?.GetValueKind() is not JsonValueKind.False) + { + path?.Add(AdditionalPropertiesPropertyName); + schemaObj[AdditionalPropertiesPropertyName] = TransformSchemaCore(additionalProps, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(NotPropertyName, out JsonNode? notSchema)) + { + path?.Add(NotPropertyName); + schemaObj[NotPropertyName] = TransformSchemaCore(notSchema, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + // Traverse keywords that contain arrays of schemas + ReadOnlySpan combinatorKeywords = ["anyOf", "oneOf", "allOf"]; + foreach (string combinatorKeyword in combinatorKeywords) + { + if (schemaObj.TryGetPropertyValue(combinatorKeyword, out JsonNode? combinatorSchema) && combinatorSchema is JsonArray combinatorArray) + { + path?.Add(combinatorKeyword); + for (int i = 0; i < combinatorArray.Count; i++) + { + path?.Add($"[{i}]"); + JsonNode element = TransformSchemaCore(combinatorArray[i], transformOptions, path); + if (!ReferenceEquals(element, combinatorArray[i])) + { + combinatorArray[i] = element; + } + + path?.RemoveAt(path.Count - 1); + } + + path?.RemoveAt(path.Count - 1); + } + } + + // Step 2. Apply node-level transformations per the settings. + if (transformOptions.DisallowAdditionalProperties && properties is not null && !schemaObj.ContainsKey(AdditionalPropertiesPropertyName)) + { + schemaObj[AdditionalPropertiesPropertyName] = (JsonNode)false; + } + + if (transformOptions.RequireAllProperties && properties is not null) + { + JsonArray requiredProps = []; + foreach (var prop in properties) + { + requiredProps.Add((JsonNode)prop.Key); + } + + schemaObj[RequiredPropertyName] = requiredProps; + } + + if (transformOptions.UseNullableKeyword && + schemaObj.TryGetPropertyValue(TypePropertyName, out JsonNode? typeSchema) && + typeSchema is JsonArray typeArray) + { + bool isNullable = false; + string? foundType = null; + + foreach (JsonNode? typeNode in typeArray) + { + string typeString = (string)typeNode!; + if (typeString is "null") + { + isNullable = true; + continue; + } + + if (foundType is not null) + { + // The array contains more than one non-null types, abort the transformation. + foundType = null; + break; + } + + foundType = typeString; + } + + if (isNullable && foundType is not null) + { + schemaObj["type"] = (JsonNode)foundType; + schemaObj["nullable"] = (JsonNode)true; + } + } + + break; + + default: + Throw.ArgumentException(nameof(schema), "Schema must be an object or a boolean value."); + break; + } + + // Apply user-defined transformations as the final step. + if (transformOptions.TransformSchemaNode is { } transformer) + { + Debug.Assert(path != null, "Path should not be null when TransformSchemaNode is provided."); + schema = transformer(new AIJsonSchemaTransformContext(path!.ToArray()), schema); + } + + return schema; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs index ea66cc191e4..ff62845eb0e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs @@ -24,6 +24,14 @@ namespace Microsoft.Extensions.AI; /// Represents an for an Azure AI Inference . internal sealed class AzureAIInferenceChatClient : IChatClient { + /// Gets the JSON schema transform cache conforming to OpenAI restrictions per https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas. + private static AIJsonSchemaTransformCache SchemaTransformCache { get; } = new(new() + { + RequireAllProperties = true, + DisallowAdditionalProperties = true, + ConvertBooleanSchemas = true + }); + /// Metadata about the client. private readonly ChatClientMetadata _metadata; @@ -273,66 +281,74 @@ private static ChatRole ToChatRole(global::Azure.AI.Inference.ChatRole role) => finishReason == CompletionsFinishReason.ToolCalls ? ChatFinishReason.ToolCalls : new(s); + private ChatCompletionsOptions CreateAzureAIOptions(IEnumerable chatContents, ChatOptions? options) => + new(ToAzureAIInferenceChatMessages(chatContents)) + { + Model = options?.ModelId ?? _metadata.DefaultModelId ?? + throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options.") + }; + /// Converts an extensions options instance to an AzureAI options instance. private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatContents, ChatOptions? options) { - ChatCompletionsOptions result = new(ToAzureAIInferenceChatMessages(chatContents)) + if (options is null) { - Model = options?.ModelId ?? _metadata.DefaultModelId ?? throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options.") - }; + return CreateAzureAIOptions(chatContents, options); + } - if (options is not null) + if (options.RawRepresentationFactory?.Invoke(this) is ChatCompletionsOptions result) + { + result.Messages = ToAzureAIInferenceChatMessages(chatContents).ToList(); + result.Model ??= options.ModelId ?? _metadata.DefaultModelId ?? + throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options."); + } + else { - result.FrequencyPenalty = options.FrequencyPenalty; - result.MaxTokens = options.MaxOutputTokens; - result.NucleusSamplingFactor = options.TopP; - result.PresencePenalty = options.PresencePenalty; - result.Temperature = options.Temperature; - result.Seed = options.Seed; - - if (options.StopSequences is { Count: > 0 } stopSequences) + result = CreateAzureAIOptions(chatContents, options); + } + + result.FrequencyPenalty ??= options.FrequencyPenalty; + result.MaxTokens ??= options.MaxOutputTokens; + result.NucleusSamplingFactor ??= options.TopP; + result.PresencePenalty ??= options.PresencePenalty; + result.Temperature ??= options.Temperature; + result.Seed ??= options.Seed; + + if (options.StopSequences is { Count: > 0 } stopSequences) + { + foreach (string stopSequence in stopSequences) { - foreach (string stopSequence in stopSequences) - { - result.StopSequences.Add(stopSequence); - } + result.StopSequences.Add(stopSequence); } + } + + // This property is strongly typed on ChatOptions but not on ChatCompletionsOptions. + if (options.TopK is int topK && !result.AdditionalProperties.ContainsKey("top_k")) + { + result.AdditionalProperties["top_k"] = new BinaryData(JsonSerializer.SerializeToUtf8Bytes(topK, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(int)))); + } - // These properties are strongly typed on ChatOptions but not on ChatCompletionsOptions. - if (options.TopK is int topK) + if (options.AdditionalProperties is { } props) + { + foreach (var prop in props) { - result.AdditionalProperties["top_k"] = new BinaryData(JsonSerializer.SerializeToUtf8Bytes(topK, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(int)))); + byte[] data = JsonSerializer.SerializeToUtf8Bytes(prop.Value, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); + result.AdditionalProperties[prop.Key] = new BinaryData(data); } + } - if (options.AdditionalProperties is { } props) + if (options.Tools is { Count: > 0 } tools) + { + foreach (AITool tool in tools) { - foreach (var prop in props) + if (tool is AIFunction af) { - switch (prop.Key) - { - // Propagate everything else to the ChatCompletionsOptions' AdditionalProperties. - default: - if (prop.Value is not null) - { - byte[] data = JsonSerializer.SerializeToUtf8Bytes(prop.Value, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); - result.AdditionalProperties[prop.Key] = new BinaryData(data); - } - - break; - } + result.Tools.Add(ToAzureAIChatTool(af)); } } - if (options.Tools is { Count: > 0 } tools) + if (result.ToolChoice is null && result.Tools.Count > 0) { - foreach (AITool tool in tools) - { - if (tool is AIFunction af) - { - result.Tools.Add(ToAzureAIChatTool(af)); - } - } - switch (options.ToolMode) { case NoneChatToolMode: @@ -351,14 +367,17 @@ private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatCon break; } } + } + if (result.ResponseFormat is null) + { if (options.ResponseFormat is ChatResponseFormatText) { result.ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat(); } else if (options.ResponseFormat is ChatResponseFormatJson json) { - if (json.Schema is { } schema) + if (SchemaTransformCache.GetOrCreateTransformedSchema(json) is { } schema) { var tool = JsonSerializer.Deserialize(schema, JsonContext.Default.AzureAIChatToolJson)!; result.ResponseFormat = ChatCompletionsResponseFormat.CreateJsonFormat( @@ -392,7 +411,7 @@ private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatCon private static ChatCompletionsToolDefinition ToAzureAIChatTool(AIFunction aiFunction) { // Map to an intermediate model so that redundant properties are skipped. - var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema, JsonContext.Default.AzureAIChatToolJson)!; + var tool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(aiFunction), JsonContext.Default.AzureAIChatToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool, JsonContext.Default.AzureAIChatToolJson)); return new(new FunctionDefinition(aiFunction.Name) { diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceEmbeddingGenerator.cs index 9721befca0b..d96112c73c6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceEmbeddingGenerator.cs @@ -89,6 +89,8 @@ public AzureAIInferenceEmbeddingGenerator( public async Task>> GenerateAsync( IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) { + _ = Throw.IfNull(values); + var azureAIOptions = ToAzureAIOptions(values, options, EmbeddingEncodingFormat.Base64); var embeddings = (await _embeddingsClient.EmbedAsync(azureAIOptions, cancellationToken).ConfigureAwait(false)).Value; @@ -118,7 +120,7 @@ void IDisposable.Dispose() // Nothing to dispose. Implementation required for the IEmbeddingGenerator interface. } - private static float[] ParseBase64Floats(BinaryData binaryData) + internal static float[] ParseBase64Floats(BinaryData binaryData) { ReadOnlySpan base64 = binaryData.ToMemory().Span; @@ -161,7 +163,7 @@ static void ThrowInvalidData() => throw new FormatException("The input is not a valid Base64 string of encoded floats."); } - /// Converts an extensions options instance to an OpenAI options instance. + /// Converts an extensions options instance to an Azure.AI.Inference options instance. private EmbeddingsOptions ToAzureAIOptions(IEnumerable inputs, EmbeddingGenerationOptions? options, EmbeddingEncodingFormat format) { EmbeddingsOptions result = new(inputs) diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceExtensions.cs index 58739b00b0d..40d2932dd08 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceExtensions.cs @@ -24,4 +24,13 @@ public static IChatClient AsIChatClient( public static IEmbeddingGenerator> AsIEmbeddingGenerator( this EmbeddingsClient embeddingsClient, string? defaultModelId = null, int? defaultModelDimensions = null) => new AzureAIInferenceEmbeddingGenerator(embeddingsClient, defaultModelId, defaultModelDimensions); + + /// Gets an for use with this . + /// The client. + /// The ID of the model to use. If , it can be provided per request via . + /// The number of dimensions generated in each embedding. + /// An that can be used to generate embeddings via the . + public static IEmbeddingGenerator> AsIEmbeddingGenerator( + this ImageEmbeddingsClient imageEmbeddingsClient, string? defaultModelId = null, int? defaultModelDimensions = null) => + new AzureAIInferenceImageEmbeddingGenerator(imageEmbeddingsClient, defaultModelId, defaultModelDimensions); } diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceImageEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceImageEmbeddingGenerator.cs new file mode 100644 index 00000000000..ec6ee9dd6e6 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceImageEmbeddingGenerator.cs @@ -0,0 +1,143 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.Inference; +using Microsoft.Shared.Diagnostics; + +#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test +#pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields +#pragma warning disable S109 // Magic numbers should not be used + +namespace Microsoft.Extensions.AI; + +/// Represents an for an Azure.AI.Inference . +internal sealed class AzureAIInferenceImageEmbeddingGenerator : + IEmbeddingGenerator> +{ + /// Metadata about the embedding generator. + private readonly EmbeddingGeneratorMetadata _metadata; + + /// The underlying . + private readonly ImageEmbeddingsClient _imageEmbeddingsClient; + + /// The number of dimensions produced by the generator. + private readonly int? _dimensions; + + /// Initializes a new instance of the class. + /// The underlying client. + /// + /// The ID of the model to use. This can also be overridden per request via . + /// Either this parameter or must provide a valid model ID. + /// + /// The number of dimensions to generate in each embedding. + /// is . + /// is empty or composed entirely of whitespace. + /// is not positive. + public AzureAIInferenceImageEmbeddingGenerator( + ImageEmbeddingsClient imageEmbeddingsClient, string? defaultModelId = null, int? defaultModelDimensions = null) + { + _ = Throw.IfNull(imageEmbeddingsClient); + + if (defaultModelId is not null) + { + _ = Throw.IfNullOrWhitespace(defaultModelId); + } + + if (defaultModelDimensions is < 1) + { + Throw.ArgumentOutOfRangeException(nameof(defaultModelDimensions), "Value must be greater than 0."); + } + + _imageEmbeddingsClient = imageEmbeddingsClient; + _dimensions = defaultModelDimensions; + + // https://github.com/Azure/azure-sdk-for-net/issues/46278 + // The endpoint isn't currently exposed, so use reflection to get at it, temporarily. Once packages + // implement the abstractions directly rather than providing adapters on top of the public APIs, + // the package can provide such implementations separate from what's exposed in the public API. + var providerUrl = typeof(ImageEmbeddingsClient).GetField("_endpoint", BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance) + ?.GetValue(imageEmbeddingsClient) as Uri; + + _metadata = new EmbeddingGeneratorMetadata("az.ai.inference", providerUrl, defaultModelId, defaultModelDimensions); + } + + /// + object? IEmbeddingGenerator.GetService(Type serviceType, object? serviceKey) + { + _ = Throw.IfNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(ImageEmbeddingsClient) ? _imageEmbeddingsClient : + serviceType == typeof(EmbeddingGeneratorMetadata) ? _metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } + + /// + public async Task>> GenerateAsync( + IEnumerable values, EmbeddingGenerationOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(values); + + var azureAIOptions = ToAzureAIOptions(values, options, EmbeddingEncodingFormat.Base64); + + var embeddings = (await _imageEmbeddingsClient.EmbedAsync(azureAIOptions, cancellationToken).ConfigureAwait(false)).Value; + + GeneratedEmbeddings> result = new(embeddings.Data.Select(e => + new Embedding(AzureAIInferenceEmbeddingGenerator.ParseBase64Floats(e.Embedding)) + { + CreatedAt = DateTimeOffset.UtcNow, + ModelId = embeddings.Model ?? azureAIOptions.Model, + })); + + if (embeddings.Usage is not null) + { + result.Usage = new() + { + InputTokenCount = embeddings.Usage.PromptTokens, + TotalTokenCount = embeddings.Usage.TotalTokens + }; + } + + return result; + } + + /// + void IDisposable.Dispose() + { + // Nothing to dispose. Implementation required for the IEmbeddingGenerator interface. + } + + /// Converts an extensions options instance to an Azure.AI.Inference options instance. + private ImageEmbeddingsOptions ToAzureAIOptions(IEnumerable inputs, EmbeddingGenerationOptions? options, EmbeddingEncodingFormat format) + { + ImageEmbeddingsOptions result = new(inputs.Select(dc => new ImageEmbeddingInput(dc.Uri))) + { + Dimensions = options?.Dimensions ?? _dimensions, + Model = options?.ModelId ?? _metadata.DefaultModelId, + EncodingFormat = format, + }; + + if (options?.AdditionalProperties is { } props) + { + foreach (var prop in props) + { + if (prop.Value is not null) + { + byte[] data = JsonSerializer.SerializeToUtf8Bytes(prop.Value, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); + result.AdditionalProperties[prop.Key] = new BinaryData(data); + } + } + } + + return result; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md index a5c9ceb0a42..aaf1ac1c67c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## 9.4.3-preview.1.25230.7 + +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.0-preview.1.25207.5 - Updated to Azure.AI.Inference 1.0.0-beta.4. @@ -36,4 +40,4 @@ ## 9.0.0-preview.9.24507.7 -Initial Preview +- Initial Preview diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index 08f035d55eb..b0d975edb43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -18,7 +18,7 @@ internal sealed class CleanCacheCommand(ILogger logger) { internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) { - IResponseCacheProvider cacheProvider; + IEvaluationResponseCacheProvider cacheProvider; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 59635dc0530..8d6617d8302 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -23,7 +23,7 @@ internal async Task InvokeAsync( int lastN, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index f2466923bd8..2611695e542 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -28,7 +28,7 @@ internal async Task InvokeAsync( Format format, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj index 5980faefe72..a1d9252e8e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj @@ -1,11 +1,9 @@  - A dotnet tool for managing the evaluation data and generating reports. + A command line dotnet tool for generating reports and managing evaluation data. Exe - - $(MinimumSupportedTfmForPackaging) + $(NetCoreTargetFrameworks) Microsoft.Extensions.AI.Evaluation.Console $(NoWarn);EA0000 @@ -22,6 +20,15 @@ 0 + + + false + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md index ba6f61c446a..c21e2a299ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj index ef591150f95..07771569ba0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Microsoft.Extensions.AI.Evaluation.Quality.csproj @@ -1,7 +1,7 @@  - A library containing a set of evaluators for evaluating the quality (coherence, relevance, truth, completeness, groundedness, fluency, equivalence etc.) of responses received from an LLM. + A library containing evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. $(TargetFrameworks);netstandard2.0 Microsoft.Extensions.AI.Evaluation.Quality diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/README.md index ba6f61c446a..c21e2a299ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs deleted file mode 100644 index 2ec6cdb801f..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageCamelCaseEnumConverter() : - JsonStringEnumConverter(JsonNamingPolicy.CamelCase) - where TEnum : struct, System.Enum; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs index 23b2ae9c88c..b36c8d8bd56 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -11,7 +12,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class AzureStorageJsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -24,6 +25,7 @@ internal static class Compact { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); + internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -45,14 +47,14 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( Converters = [ - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageTimeSpanConverter) + typeof(CamelCaseEnumConverter), + typeof(CamelCaseEnumConverter), + typeof(TimeSpanConverter), + typeof(EvaluationContextConverter) ], WriteIndented = true, IgnoreReadOnlyProperties = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] private sealed partial class JsonContext : JsonSerializerContext; - } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs deleted file mode 100644 index 0c064ededd3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageTimeSpanConverter : JsonConverter -{ - public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - => TimeSpan.FromSeconds(reader.GetDouble()); - - public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) - => writer.WriteNumberValue(value.TotalSeconds); -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj index f705add750e..237df014d0d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj @@ -1,7 +1,7 @@  - A library that provides additional an additional storage provider based on Azure Storage containers. + A library that supports the Microsoft.Extensions.AI.Evaluation.Reporting library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. $(TargetFrameworks);netstandard2.0 Microsoft.Extensions.AI.Evaluation.Reporting @@ -17,6 +17,12 @@ 0 + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/README.md index ba6f61c446a..c21e2a299ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs index 9302107b926..fafd8639b34 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs @@ -24,10 +24,6 @@ public static class AzureStorageReportingConfiguration /// /// The set of s that should be invoked to evaluate AI responses. /// - /// - /// An optional that specifies the maximum amount of time that cached AI responses should - /// survive in the cache before they are considered expired and evicted. - /// /// /// A that specifies the that is used by AI-based /// included in the returned . Can be omitted if @@ -36,6 +32,10 @@ public static class AzureStorageReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -63,21 +63,21 @@ public static class AzureStorageReportingConfiguration public static ReportingConfiguration Create( DataLakeDirectoryClient client, IEnumerable evaluators, - TimeSpan? timeToLiveForCacheEntries = null, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, IEnumerable? tags = null) #pragma warning restore S107 { - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching ? new AzureStorageResponseCacheProvider(client, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new AzureStorageResultStore(client); + IEvaluationResultStore resultStore = new AzureStorageResultStore(client); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs index f7115a37024..5e83b456a0b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs @@ -40,9 +40,9 @@ internal sealed partial class AzureStorageResponseCache( private const string EntryAndContentsFilesNotFound = "Cache entry file {0} and contents file {1} were not found."; private readonly string _iterationPath = $"cache/{scenarioName}/{iterationName}"; + private readonly Func _provideDateTime = provideDateTime; private readonly TimeSpan _timeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; - private readonly Func _provideDateTime = provideDateTime; public byte[]? Get(string key) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs index a890fa80332..6c6d1431a1a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs @@ -15,8 +15,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under an Azure Storage container. +/// An that returns an that can cache AI +/// responses for a particular under an Azure Storage container. /// /// /// A with access to an Azure Storage container under which the cached AI @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class AzureStorageResponseCacheProvider( DataLakeDirectoryClient client, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.Now; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs index 70d988abe74..71682f13651 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs @@ -20,14 +20,14 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s under an Azure Storage -/// container. +/// An implementation that stores s under an Azure +/// Storage container. /// /// /// A with access to an Azure Storage container under which the /// s should be stored. /// -public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IResultStore +public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IEvaluationResultStore { private const string ResultsRootPrefix = "results"; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs index 006dfc741e8..71afe53217a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Evaluation.Reporting; @@ -11,19 +12,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; public static class ChatDetailsExtensions { /// - /// Adds for a particular LLM chat conversation turn to the + /// Adds for one or more LLM chat conversation turns to the /// collection. /// /// - /// The object to which the is to be added. + /// The object to which the are to be added. /// /// - /// The for a particular LLM chat conversation turn. + /// The for one or more LLM chat conversation turns. /// - public static void AddTurnDetails(this ChatDetails chatDetails, ChatTurnDetails turnDetails) + public static void AddTurnDetails(this ChatDetails chatDetails, IEnumerable turnDetails) { _ = Throw.IfNull(chatDetails); + _ = Throw.IfNull(turnDetails); - chatDetails.TurnDetails.Add(turnDetails); + foreach (ChatTurnDetails t in turnDetails) + { + chatDetails.TurnDetails.Add(t); + } } + + /// + /// Adds for one or more LLM chat conversation turns to the + /// collection. + /// + /// + /// The object to which the are to be added. + /// + /// + /// The for one or more LLM chat conversation turns. + /// + public static void AddTurnDetails(this ChatDetails chatDetails, params ChatTurnDetails[] turnDetails) + => chatDetails.AddTurnDetails(turnDetails as IEnumerable); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs index e5cd6b26ec3..f25fa074430 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs @@ -27,13 +27,11 @@ public static class Defaults /// /// Gets a that specifies the default amount of time that cached AI responses should survive - /// in the 's cache before they are considered expired and evicted. + /// in the 's cache before they are considered expired and evicted. /// public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14); - /// - /// Defines the version number for the reporting format. If and when the serialized format undergoes - /// breaking changes, this number will be incremented. - /// + // Defines the version number for the reporting format. If and when the serialized format undergoes + // breaking changes, this number should be incremented. internal const int ReportingFormatVersion = 1; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs similarity index 58% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs index 6bc8ce25432..1859124a98f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs @@ -12,26 +12,28 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// . /// /// -/// can be used to set up caching of AI-generated responses (both the AI responses -/// under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, the AI -/// responses associated with each are stored in the that is -/// returned from this . So long as the inputs (such as the content included in the -/// requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the same -/// use the cached responses instead of invoking the AI model to generate new ones. Bypassing -/// the AI model when the inputs remain unchanged results in faster execution at a lower cost. +/// can be used to set up caching of AI-generated responses (both the AI +/// responses under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, +/// the AI responses associated with each are stored in the +/// that is returned from this . So long as the inputs (such as the +/// content included in the requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the +/// same use the cached responses instead of invoking the AI model to generate new ones. +/// Bypassing the AI model when the inputs remain unchanged results in faster execution at a lower cost. /// -public interface IResponseCacheProvider +public interface IEvaluationResponseCacheProvider { /// - /// Returns an that caches the AI responses associated with a particular - /// . + /// Returns an that caches all the AI responses associated with the + /// with the supplied and + /// . /// /// The . /// The . /// A that can cancel the operation. /// - /// An that caches the AI responses associated with a particular - /// . + /// An that caches all the AI responses associated with the + /// with the supplied and + /// . /// ValueTask GetCacheAsync( string scenarioName, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs index 3f3dea6cc7a..202a6305cd3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// /// Represents a store for s. /// -public interface IResultStore +public interface IEvaluationResultStore { /// /// Returns s for s filtered by the specified diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs index e372b7e8434..3a8c2af1ce2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -12,14 +13,13 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class JsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: true); internal static JsonTypeInfo DatasetTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); - internal static JsonTypeInfo CacheOptionsTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -29,7 +29,6 @@ internal static class Compact internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); internal static JsonTypeInfo DatasetTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); - internal static JsonTypeInfo CacheOptionsTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -47,15 +46,13 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden return options; } - [JsonSerializable(typeof(EvaluationResult))] + [JsonSerializable(typeof(ScenarioRunResult))] [JsonSerializable(typeof(Dataset))] [JsonSerializable(typeof(CacheEntry))] - [JsonSerializable(typeof(CacheOptions))] [JsonSourceGenerationOptions( Converters = [ typeof(CamelCaseEnumConverter), typeof(CamelCaseEnumConverter), - typeof(CamelCaseEnumConverter), typeof(TimeSpanConverter), typeof(EvaluationContextConverter) ], diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj index 1a920c47e8d..a06db14fffd 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Microsoft.Extensions.AI.Evaluation.Reporting.csproj @@ -8,7 +8,7 @@ --> - A library for aggregating and reporting evaluation data. This library also includes support for caching LLM responses. + A library that contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. $(TargetFrameworks);netstandard2.0 Microsoft.Extensions.AI.Evaluation.Reporting diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/README.md index ba6f61c446a..c21e2a299ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs index 68a73338d88..130586de930 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs @@ -27,9 +27,10 @@ public sealed class ReportingConfiguration public IReadOnlyList Evaluators { get; } /// - /// Gets the that should be used to persist the s. + /// Gets the that should be used to persist the + /// s. /// - public IResultStore ResultStore { get; } + public IEvaluationResultStore ResultStore { get; } /// /// Gets a that specifies the that is used by @@ -38,9 +39,9 @@ public sealed class ReportingConfiguration public ChatConfiguration? ChatConfiguration { get; } /// - /// Gets the that should be used to cache AI responses. + /// Gets the that should be used to cache AI responses. /// - public IResponseCacheProvider? ResponseCacheProvider { get; } + public IEvaluationResponseCacheProvider? ResponseCacheProvider { get; } /// /// Gets the collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -101,7 +102,7 @@ public sealed class ReportingConfiguration /// The set of s that should be invoked to evaluate AI responses. /// /// - /// The that should be used to persist the s. + /// The that should be used to persist the s. /// /// /// A that specifies the that is used by @@ -109,8 +110,8 @@ public sealed class ReportingConfiguration /// none of the included are AI-based. /// /// - /// The that should be used to cache AI responses. If omitted, AI responses - /// will not be cached. + /// The that should be used to cache AI responses. If omitted, AI + /// responses will not be cached. /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -134,9 +135,9 @@ public sealed class ReportingConfiguration #pragma warning disable S107 // Methods should not have too many parameters public ReportingConfiguration( IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, - IResponseCacheProvider? responseCacheProvider = null, + IEvaluationResponseCacheProvider? responseCacheProvider = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs index eb58685bf0c..5fa46e7e4ec 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs @@ -93,7 +93,7 @@ public sealed class ScenarioRun : IAsyncDisposable public ChatConfiguration? ChatConfiguration { get; } private readonly CompositeEvaluator _compositeEvaluator; - private readonly IResultStore _resultStore; + private readonly IEvaluationResultStore _resultStore; private readonly Func? _evaluationMetricInterpreter; private readonly ChatDetails? _chatDetails; private readonly IEnumerable? _tags; @@ -106,7 +106,7 @@ internal ScenarioRun( string iterationName, string executionName, IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, Func? evaluationMetricInterpreter = null, ChatDetails? chatDetails = null, @@ -189,7 +189,7 @@ await _compositeEvaluator.EvaluateAsync( /// /// Disposes the and writes the to the configured - /// . + /// . /// /// A that represents the asynchronous operation. public async ValueTask DisposeAsync() diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs index 3b723a2d258..08822f18d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs @@ -85,16 +85,11 @@ public static ValueTask EvaluateAsync( this ScenarioRun scenarioRun, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -148,16 +143,12 @@ public static ValueTask EvaluateAsync( ChatMessage userRequest, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs index 94ab92e177b..e967fdd1db9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs @@ -32,6 +32,10 @@ public static class DiskBasedReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -61,6 +65,7 @@ public static ReportingConfiguration Create( IEnumerable evaluators, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, @@ -69,12 +74,12 @@ public static ReportingConfiguration Create( { storageRootPath = Path.GetFullPath(storageRootPath); - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching - ? new DiskBasedResponseCacheProvider(storageRootPath) + ? new DiskBasedResponseCacheProvider(storageRootPath, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new DiskBasedResultStore(storageRootPath); + IEvaluationResultStore resultStore = new DiskBasedResultStore(storageRootPath); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs deleted file mode 100644 index cbc8ca2f151..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; - -internal partial class DiskBasedResponseCache -{ - /// - /// An enum representing the mode in which the cache is operating. - /// - internal enum CacheMode - { - /// - /// In this mode, the cache is disabled. All requests bypass the cache and are forwarded online. - /// - Disabled, - - /// - /// In this mode, the cache is enabled. Requests are handled by the cache first. If a cached response is not - /// available, then the request is forwarded online. - /// - Enabled, - - /// - /// In this mode, the cache is enabled. However, requests are never forwarded online. Instead if a cached response - /// is not available, then an exception is thrown. Additionally in this mode, the cache is considered frozen (or - /// read only) which means that all the cache artifacts (including expired entries) are preserved as is on disk. - /// - EnabledOfflineOnly - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs deleted file mode 100644 index 1e21c59828b..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Globalization; -using System.IO; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; - -internal partial class DiskBasedResponseCache -{ - internal sealed class CacheOptions - { - public static CacheOptions Default { get; } = new CacheOptions(); - - private const string DeserializationFailedMessage = "Unable to deserialize the cache options file at {0}."; - - public CacheOptions(CacheMode mode = CacheMode.Enabled, TimeSpan? timeToLiveForCacheEntries = null) - { - Mode = mode; - TimeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; - } - - [JsonConstructor] - public CacheOptions(CacheMode mode, TimeSpan timeToLiveForCacheEntries) - { - Mode = mode; - TimeToLiveForCacheEntries = timeToLiveForCacheEntries; - } - - public CacheMode Mode { get; } - - [JsonPropertyName("timeToLiveInSecondsForCacheEntries")] - public TimeSpan TimeToLiveForCacheEntries { get; } - - public static CacheOptions Read(string cacheOptionsFilePath) - { - using FileStream cacheOptionsFile = File.OpenRead(cacheOptionsFilePath); - - CacheOptions cacheOptions = - JsonSerializer.Deserialize( - cacheOptionsFile, - JsonUtilities.Default.CacheOptionsTypeInfo) ?? - throw new JsonException( - string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, cacheOptionsFilePath)); - - return cacheOptions; - } - - public static async Task ReadAsync( - string cacheOptionsFilePath, - CancellationToken cancellationToken = default) - { - using FileStream cacheOptionsFile = File.OpenRead(cacheOptionsFilePath); - - CacheOptions cacheOptions = - await JsonSerializer.DeserializeAsync( - cacheOptionsFile, - JsonUtilities.Default.CacheOptionsTypeInfo, - cancellationToken).ConfigureAwait(false) ?? - throw new JsonException( - string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, cacheOptionsFilePath)); - - return cacheOptions; - } - - public void Write(string cacheOptionsFilePath) - { - using FileStream cacheOptionsFile = File.Create(cacheOptionsFilePath); - JsonSerializer.Serialize(cacheOptionsFile, this, JsonUtilities.Default.CacheOptionsTypeInfo); - } - - public async Task WriteAsync( - string cacheOptionsFilePath, - CancellationToken cancellationToken = default) - { - using FileStream cacheOptionsFile = File.Create(cacheOptionsFilePath); - await JsonSerializer.SerializeAsync( - cacheOptionsFile, - this, - JsonUtilities.Default.CacheOptionsTypeInfo, - cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs index 51b96739964..d0a107d8710 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs @@ -1,6 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable S3604 +// S3604: Member initializer values should not be redundant. +// We disable this warning because it is a false positive arising from the analyzer's lack of support for C#'s primary +// constructor syntax. + #pragma warning disable CA1725 // CA1725: Parameter names should match base declaration. // All functions on 'IDistributedCache' use the parameter name 'token' in place of 'cancellationToken'. However, @@ -26,55 +31,42 @@ internal sealed partial class DiskBasedResponseCache : IDistributedCache private readonly string _scenarioName; private readonly string _iterationName; - private readonly CacheOptions _options; private readonly string _iterationPath; private readonly Func _provideDateTime; + private readonly TimeSpan _timeToLiveForCacheEntries; - public DiskBasedResponseCache( + internal DiskBasedResponseCache( string storageRootPath, string scenarioName, string iterationName, - Func provideDateTime) + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) { _scenarioName = scenarioName; _iterationName = iterationName; storageRootPath = Path.GetFullPath(storageRootPath); string cacheRootPath = GetCacheRootPath(storageRootPath); - string optionsFilePath = GetOptionsFilePath(cacheRootPath); - _options = File.Exists(optionsFilePath) ? CacheOptions.Read(optionsFilePath) : CacheOptions.Default; + _iterationPath = Path.Combine(cacheRootPath, scenarioName, iterationName); _provideDateTime = provideDateTime; + _timeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; } public byte[]? Get(string key) { - if (_options.Mode is CacheMode.Disabled) - { - return null; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { - return _options.Mode is CacheMode.EnabledOfflineOnly - ? throw new FileNotFoundException( - string.Format( - CultureInfo.CurrentCulture, - EntryAndContentsFilesNotFound, - entryFilePath, - contentsFilePath)) - : null; + return null; } - if (_options.Mode is not CacheMode.EnabledOfflineOnly) + CacheEntry entry = CacheEntry.Read(entryFilePath); + if (entry.Expiration <= _provideDateTime()) { - CacheEntry entry = CacheEntry.Read(entryFilePath); - if (entry.Expiration <= _provideDateTime()) - { - Remove(key); - return null; - } + Remove(key); + return null; } return File.ReadAllBytes(contentsFilePath); @@ -82,34 +74,20 @@ public DiskBasedResponseCache( public async Task GetAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled) - { - return null; - } - (string _, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { - return _options.Mode is CacheMode.EnabledOfflineOnly - ? throw new FileNotFoundException( - string.Format( - CultureInfo.CurrentCulture, - EntryAndContentsFilesNotFound, - entryFilePath, - contentsFilePath)) - : null; + return null; } - if (_options.Mode is not CacheMode.EnabledOfflineOnly) - { - CacheEntry entry = - await CacheEntry.ReadAsync(entryFilePath, cancellationToken: cancellationToken).ConfigureAwait(false); + CacheEntry entry = + await CacheEntry.ReadAsync(entryFilePath, cancellationToken: cancellationToken).ConfigureAwait(false); - if (entry.Expiration <= _provideDateTime()) - { - await RemoveAsync(key, cancellationToken).ConfigureAwait(false); - return null; - } + if (entry.Expiration <= _provideDateTime()) + { + await RemoveAsync(key, cancellationToken).ConfigureAwait(false); + return null; } #if NET @@ -162,12 +140,8 @@ await stream.ReadAsync( public void Refresh(string key) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { throw new FileNotFoundException( @@ -184,12 +158,8 @@ public void Refresh(string key) public async Task RefreshAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { throw new FileNotFoundException( @@ -206,33 +176,20 @@ public async Task RefreshAsync(string key, CancellationToken cancellationToken = public void Remove(string key) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, _, _, _) = GetPaths(key); + Directory.Delete(keyPath, recursive: true); } public Task RemoveAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return Task.CompletedTask; - } - Remove(key); + return Task.CompletedTask; } public void Set(string key, byte[] value, DistributedCacheEntryOptions options) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, string entryFilePath, string contentsFilePath, _) = GetPaths(key); _ = Directory.CreateDirectory(keyPath); @@ -249,11 +206,6 @@ public async Task SetAsync( DistributedCacheEntryOptions options, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, string entryFilePath, string contentsFilePath, _) = GetPaths(key); Directory.CreateDirectory(keyPath); @@ -334,9 +286,6 @@ await CacheEntry.ReadAsync( private static string GetCacheRootPath(string storageRootPath) => Path.Combine(storageRootPath, "cache"); - private static string GetOptionsFilePath(string cacheRootPath) - => Path.Combine(cacheRootPath, "options.json"); - private static string GetEntryFilePath(string keyPath) => Path.Combine(keyPath, "entry.json"); @@ -368,7 +317,7 @@ private static string GetContentsFilePath(string keyPath) private CacheEntry CreateEntry() { DateTime creation = _provideDateTime(); - DateTime expiration = creation.Add(_options.TimeToLiveForCacheEntries); + DateTime expiration = creation.Add(_timeToLiveForCacheEntries); return new CacheEntry(_scenarioName, _iterationName, creation, expiration); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs index d3fce0e5aff..8b60fe5a272 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs @@ -14,21 +14,31 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under the specified on disk. +/// An that returns an that can cache +/// AI responses for a particular under the specified on +/// disk. /// /// /// The path to a directory on disk under which the cached AI responses should be stored. /// -public sealed class DiskBasedResponseCacheProvider(string storageRootPath) : IResponseCacheProvider +/// +/// An optional that specifies the maximum amount of time that cached AI responses should +/// survive in the cache before they are considered expired and evicted. +/// +public sealed class DiskBasedResponseCacheProvider( + string storageRootPath, + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.UtcNow; /// /// Intended for testing purposes only. /// - internal DiskBasedResponseCacheProvider(string storageRootPath, Func provideDateTime) - : this(storageRootPath) + internal DiskBasedResponseCacheProvider( + string storageRootPath, + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) + : this(storageRootPath, timeToLiveForCacheEntries) { _provideDateTime = provideDateTime; } @@ -39,7 +49,13 @@ public ValueTask GetCacheAsync( string iterationName, CancellationToken cancellationToken = default) { - var cache = new DiskBasedResponseCache(storageRootPath, scenarioName, iterationName, _provideDateTime); + var cache = + new DiskBasedResponseCache( + storageRootPath, + scenarioName, + iterationName, + _provideDateTime, + timeToLiveForCacheEntries); return new ValueTask(cache); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs index de1517dca99..4662857ec59 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs @@ -16,9 +16,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s on disk. +/// An implementation that stores s on disk. /// -public sealed class DiskBasedResultStore : IResultStore +public sealed class DiskBasedResultStore : IEvaluationResultStore { private const string DeserializationFailedMessage = "Unable to deserialize the scenario run result file at {0}."; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html index 8169711aca6..7f6e82be184 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html @@ -8,8 +8,7 @@ - + AI Evaluation Report diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json index 3e77d76226f..6e76e88aa03 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json @@ -33,7 +33,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } }, @@ -8274,6 +8274,51 @@ "node": "*" } }, + "node_modules/tinyglobby": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.13.tgz", + "integrity": "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.4.4", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/tinytim": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tinytim/-/tinytim-0.1.1.tgz", @@ -8764,14 +8809,18 @@ } }, "node_modules/vite": { - "version": "6.2.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", - "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz", + "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==", "dev": true, + "license": "MIT", "dependencies": { "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", "postcss": "^8.5.3", - "rollup": "^4.30.1" + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" }, "bin": { "vite": "bin/vite.js" @@ -8850,6 +8899,34 @@ "vite": "^5.4.11 || ^6.0.0" } }, + "node_modules/vite/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/walkdir": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.0.11.tgz", diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json index c2505b5d87a..0e32f4ae6f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json @@ -34,7 +34,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 4e33c64c305..0334d6aa08c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -4,6 +4,7 @@ using System; namespace Microsoft.Extensions.AI.Evaluation.Safety; + internal static class AIContentExtensions { internal static bool IsTextOrUsage(this AIContent content) @@ -13,22 +14,6 @@ internal static bool IsImageWithSupportedFormat(this AIContent content) => (content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) || (content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType)); - internal static bool IsUriBase64Encoded(this DataContent dataContent) - { - ReadOnlyMemory uri = dataContent.Uri.AsMemory(); - - int commaIndex = uri.Span.IndexOf(','); - if (commaIndex == -1) - { - return false; - } - - ReadOnlyMemory metadata = uri.Slice(0, commaIndex); - - bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase); - return isBase64Encoded; - } - private static bool IsSupportedImageFormat(string mediaType) { // 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well. diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs index c708b6c2cd3..9ab4508b407 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/CodeVulnerabilityEvaluator.cs @@ -9,8 +9,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate code completion responses -/// produced by an AI model for the presence of vulnerable code. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate code completion +/// responses produced by an AI model for the presence of vulnerable code. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs index 5c6c77e6791..e4788bcdc81 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentHarmEvaluator.cs @@ -9,8 +9,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of a variety of harmful content such as violence, hate speech, etc. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of a variety of harmful content such as violence, hate speech, etc. /// /// /// can be used to evaluate responses for all supported content harm metrics in one @@ -22,10 +22,10 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// /// /// A optional dictionary containing the mapping from the names of the metrics that are used when communicating -/// with the Azure AI Content Safety to the s of the +/// with the Azure AI Foundry Evaluation service, to the s of the /// s returned by this . /// -/// If omitted, includes mappings for all content harm metrics that are supported by the Azure AI Content Safety +/// If omitted, includes mappings for all content harm metrics that are supported by the Azure AI Foundry Evaluation /// service. This includes , /// , and /// . diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs index f6c2f028b81..69b47670935 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs @@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal sealed class ContentSafetyChatClient : IChatClient { - private const string Moniker = "Azure AI Content Safety"; + private const string Moniker = "Azure AI Foundry Evaluation"; private readonly ContentSafetyService _service; private readonly IChatClient? _originalChatClient; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs index b42631c3c06..afe90b0ac1d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyEvaluator.cs @@ -18,17 +18,17 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// /// An base class that can be used to implement s that utilize the -/// Azure AI Content Safety service to evaluate responses produced by an AI model for the presence of a variety of +/// Azure AI Foundry Evaluation service to evaluate responses produced by an AI model for the presence of a variety of /// unsafe content such as protected material, vulnerable code, harmful content etc. /// /// -/// The name of the annotation task that should be used when communicating with the Azure AI Content Safety service to -/// perform evaluations. +/// The name of the annotation task that should be used when communicating with the Azure AI Foundry Evaluation service +/// to perform evaluations. /// /// /// A dictionary containing the mapping from the names of the metrics that are used when communicating with the Azure -/// AI Content Safety to the s of the s returned by -/// this . +/// AI Foundry Evaluation service, to the s of the s +/// returned by this . /// #pragma warning disable S1694 // An abstract class should have both abstract and concrete methods public abstract class ContentSafetyEvaluator( @@ -58,12 +58,12 @@ public virtual ValueTask EvaluateAsync( } /// - /// Evaluates the supplied using the Azure AI Content Safety Service and returns - /// an containing one or more s. + /// Evaluates the supplied using the Azure AI Foundry Evaluation Service and + /// returns an containing one or more s. /// /// - /// The that should be used to communicate with the Azure AI Content Safety Service when - /// performing evaluations. + /// The that should be used to communicate with the Azure AI Foundry Evaluation Service + /// when performing evaluations. /// /// /// The conversation history including the request that produced the supplied . @@ -75,11 +75,11 @@ public virtual ValueTask EvaluateAsync( /// /// /// An identifier that specifies the format of the payload that should be used when communicating with the Azure AI - /// Content Safety service to perform evaluations. + /// Foundry Evaluation service to perform evaluations. /// /// /// A flag that indicates whether the names of the metrics should be included in the payload - /// that is sent to the Azure AI Content Safety service when performing evaluations. + /// that is sent to the Azure AI Foundry Evaluation service when performing evaluations. /// /// /// A that can cancel the evaluation operation. @@ -151,12 +151,13 @@ await TimingHelper.ExecuteWithTimingAsync(() => string annotationResult = annotationResponse.Text; EvaluationResult result = ContentSafetyService.ParseAnnotationResult(annotationResult); - UpdateMetrics(); + EvaluationResult updatedResult = UpdateMetrics(); + return updatedResult; - return result; - - void UpdateMetrics() + EvaluationResult UpdateMetrics() { + EvaluationResult updatedResult = new EvaluationResult(); + foreach (EvaluationMetric metric in result.Metrics.Values) { string contentSafetyServiceMetricName = metric.Name; @@ -185,7 +186,11 @@ void UpdateMetrics() // metric.LogJsonData(payload); // metric.LogJsonData(annotationResult); #pragma warning restore S125 + + updatedResult.Metrics.Add(metric.Name, metric); } + + return updatedResult; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs index 291952d761b..6028a82544c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs @@ -120,7 +120,7 @@ internal static EvaluationResult ParseAnnotationResult(string annotationResponse } } - result.Metrics[metric.Name] = metric; + result.Metrics.Add(metric.Name, metric); } return result; @@ -208,7 +208,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to retrieve discovery URL for Azure AI Content Safety service. + {evaluatorName} failed to retrieve discovery URL for Azure AI Foundry Evaluation service. {response.StatusCode} ({(int)response.StatusCode}): {response.ReasonPhrase}. To troubleshoot, see https://aka.ms/azsdk/python/evaluation/safetyevaluator/troubleshoot. """); @@ -227,7 +227,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to retrieve discovery URL from the Azure AI Content Safety service's response below. + {evaluatorName} failed to retrieve discovery URL from the Azure AI Foundry Evaluation service's response below. To troubleshoot, see https://aka.ms/azsdk/python/evaluation/safetyevaluator/troubleshoot. {responseContent} @@ -256,7 +256,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to check service availability for the Azure AI Content Safety service. + {evaluatorName} failed to check service availability for the Azure AI Foundry Evaluation service. The service is either unavailable in this region, or you lack the necessary permissions to access the AI project. {response.StatusCode} ({(int)response.StatusCode}): {response.ReasonPhrase}. To troubleshoot, see https://aka.ms/azsdk/python/evaluation/safetyevaluator/troubleshoot. @@ -283,7 +283,7 @@ await GetResponseAsync( throw new InvalidOperationException( $""" - The required {nameof(capability)} '{capability}' required for {evaluatorName} is not supported by the Azure AI Content Safety service in this region. + The required {nameof(capability)} '{capability}' required for {evaluatorName} is not supported by the Azure AI Foundry Evaluation service in this region. To troubleshoot, see https://aka.ms/azsdk/python/evaluation/safetyevaluator/troubleshoot. The following response identifies the capabilities that are supported: @@ -311,7 +311,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to submit annotation request to the Azure AI Content Safety service. + {evaluatorName} failed to submit annotation request to the Azure AI Foundry Evaluation service. {response.StatusCode} ({(int)response.StatusCode}): {response.ReasonPhrase}. To troubleshoot, see https://aka.ms/azsdk/python/evaluation/safetyevaluator/troubleshoot. """); @@ -331,7 +331,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to retrieve the result location from the following response for the annotation request submitted to The Azure AI Content Safety service. + {evaluatorName} failed to retrieve the result location from the following response for the annotation request submitted to The Azure AI Foundry Evaluation service. {responseContent} """); @@ -369,7 +369,7 @@ await GetResponseAsync( { throw new InvalidOperationException( $""" - {evaluatorName} failed to retrieve annotation result from the Azure AI Content Safety service. + {evaluatorName} failed to retrieve annotation result from the Azure AI Foundry Evaluation service. The evaluation was timed out after {elapsedDuration} seconds (and {attempts} attempts). {response.StatusCode} ({(int)response.StatusCode}): {response.ReasonPhrase}. """); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs index 615485cad5c..ec721fa59c7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs @@ -13,8 +13,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// /// Specifies configuration parameters such as the Azure AI project that should be used, and the credentials that -/// should be used, when a communicates with the Azure AI Content Safety service -/// to perform evaluations. +/// should be used, when a communicates with the Azure AI Foundry Evaluation +/// service to perform evaluations. /// /// /// The Azure that should be used when authenticating requests. @@ -29,13 +29,13 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// The name of the Azure AI project. /// /// -/// The that should be used when communicating with the Azure AI Content -/// Safety service. While the parameter is optional, it is recommended to supply an -/// that is configured with robust resilience and retry policies. +/// The that should be used when communicating with the Azure AI Foundry Evaluation service. +/// While the parameter is optional, it is recommended to supply an that is configured with +/// robust resilience and retry policies. /// /// /// The timeout (in seconds) after which a should stop retrying failed attempts -/// to communicate with the Azure AI Content Safety service when performing evaluations. +/// to communicate with the Azure AI Foundry Evaluation service when performing evaluations. /// public sealed class ContentSafetyServiceConfiguration( TokenCredential credential, @@ -66,18 +66,18 @@ public sealed class ContentSafetyServiceConfiguration( public string ProjectName { get; } = projectName; /// - /// Gets the that should be used when communicating with the Azure AI - /// Content Safety service. + /// Gets the that should be used when communicating with the Azure AI Foundry Evaluation + /// service. /// /// - /// While supplying an is optional, it is recommended to supply one that - /// is configured with robust resilience and retry policies. + /// While supplying an is optional, it is recommended to supply one that is configured + /// with robust resilience and retry policies. /// public HttpClient? HttpClient { get; } = httpClient; /// /// Gets the timeout (in seconds) after which a should stop retrying failed - /// attempts to communicate with the Azure AI Content Safety service when performing evaluations. + /// attempts to communicate with the Azure AI Foundry Evaluation service when performing evaluations. /// public int TimeoutInSecondsForRetries { get; } = timeoutInSecondsForRetries; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs index c3c316cac2e..eded31ec0f8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs @@ -11,12 +11,12 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; public static class ContentSafetyServiceConfigurationExtensions { /// - /// Returns a that can be used to communicate with the Azure AI Content Safety + /// Returns a that can be used to communicate with the Azure AI Foundry Evaluation /// service for performing content safety evaluations. /// /// /// An object that specifies configuration parameters such as the Azure AI project that should be used, and the - /// credentials that should be used, when communicating with the Azure AI Content Safety service to perform + /// credentials that should be used, when communicating with the Azure AI Foundry Evaluation service to perform /// content safety evaluations. /// /// @@ -25,11 +25,11 @@ public static class ContentSafetyServiceConfigurationExtensions /// in being replaced with /// a new that can be used both to communicate with the AI model that /// is configured to communicate with, as well as to communicate with - /// the Azure AI Content Safety service. + /// the Azure AI Foundry Evaluation service. /// /// - /// A that can be used to communicate with the Azure AI Content Safety service for - /// performing content safety evaluations. + /// A that can be used to communicate with the Azure AI Foundry Evaluation service + /// for performing content safety evaluations. /// public static ChatConfiguration ToChatConfiguration( this ContentSafetyServiceConfiguration contentSafetyServiceConfiguration, @@ -47,23 +47,23 @@ public static ChatConfiguration ToChatConfiguration( } /// - /// Returns an that can be used to communicate with the Azure AI Content Safety service - /// for performing content safety evaluations. + /// Returns an that can be used to communicate with the Azure AI Foundry Evaluation + /// service for performing content safety evaluations. /// /// /// An object that specifies configuration parameters such as the Azure AI project that should be used, and the - /// credentials that should be used, when communicating with the Azure AI Content Safety service to perform + /// credentials that should be used, when communicating with the Azure AI Foundry Evaluation service to perform /// content safety evaluations. /// /// /// The original , if any. If specified, the returned /// will be a wrapper around that can be used both /// to communicate with the AI model that is configured to communicate with, - /// as well as to communicate with the Azure AI Content Safety service. + /// as well as to communicate with the Azure AI Foundry Evaluation service. /// /// - /// A that can be used to communicate with the Azure AI Content Safety service for - /// performing content safety evaluations. + /// A that can be used to communicate with the Azure AI Foundry Evaluation service + /// for performing content safety evaluations. /// public static IChatClient ToIChatClient( this ContentSafetyServiceConfiguration contentSafetyServiceConfiguration, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs index 428940955ff..b771dc008c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs @@ -9,5 +9,5 @@ internal enum ContentSafetyServicePayloadFormat QuestionAnswer, QueryResponse, ContextCompletion, - Conversation, + Conversation } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs index a2694669106..feecec3be46 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs @@ -343,25 +343,13 @@ IEnumerable GetContents(ChatMessage message) } else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { - string url; - if (dataContent.IsUriBase64Encoded()) - { - url = dataContent.Uri; - } - else - { - BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); - string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); - url = $"data:{dataContent.MediaType};base64,{base64ImageData}"; - } - yield return new JsonObject { ["type"] = "image_url", ["image_url"] = new JsonObject { - ["url"] = url + ["url"] = dataContent.Uri } }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/GroundednessProEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/GroundednessProEvaluator.cs index 3d05f3e7056..f65ddae4662 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/GroundednessProEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/GroundednessProEvaluator.cs @@ -11,7 +11,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate the groundedness of +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate the groundedness of /// responses produced by an AI model. /// /// @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// evaluate the contents of the last conversation turn. The contents of previous conversation turns will be ignored. /// /// -/// The Azure AI Content Safety service uses a finetuned model to perform this evaluation which is expected to +/// The Azure AI Foundry Evaluation service uses a finetuned model to perform this evaluation which is expected to /// produce more accurate results than similar evaluations performed using a regular (non-finetuned) model. /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/HateAndUnfairnessEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/HateAndUnfairnessEvaluator.cs index 718b742b29a..e4f57075e34 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/HateAndUnfairnessEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/HateAndUnfairnessEvaluator.cs @@ -6,8 +6,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of content that is hateful or unfair. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of content that is hateful or unfair. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/IndirectAttackEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/IndirectAttackEvaluator.cs index f65a5ee82f6..77e7afd6e5e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/IndirectAttackEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/IndirectAttackEvaluator.cs @@ -6,8 +6,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of indirect attacks such as manipulated content, intrusion and information gathering. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of indirect attacks such as manipulated content, intrusion and information gathering. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj index 0394056245f..12512e6884c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj @@ -1,7 +1,7 @@  - A library containing a set of evaluators for evaluating the content safety (hate and unfairness, self-harm, violence etc.) of responses received from an LLM. + A library that contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. $(TargetFrameworks);netstandard2.0 Microsoft.Extensions.AI.Evaluation.Safety diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs index 25c99306c32..a1bea7ac161 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs @@ -9,8 +9,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for presence of protected material. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for presence of protected material. /// /// /// @@ -85,8 +85,9 @@ await EvaluateContentSafetyAsync( includeMetricNamesInContentSafetyServicePayload: false, cancellationToken: cancellationToken).ConfigureAwait(false); - // If images are present in the conversation, do a second evaluation for protected material in images. - // The content safety service does not support evaluating both text and images in the same request currently. + // If images are present in the conversation, do a second evaluation for protected material in images. The + // Azure AI Foundry Evaluation service does not support evaluating both text and images as part of the same + // request currently. if (messages.ContainsImageWithSupportedFormat() || modelResponse.ContainsImageWithSupportedFormat()) { EvaluationResult imageResult = @@ -100,7 +101,7 @@ await EvaluateContentSafetyAsync( foreach (EvaluationMetric imageMetric in imageResult.Metrics.Values) { - result.Metrics[imageMetric.Name] = imageMetric; + result.Metrics.Add(imageMetric.Name, imageMetric); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/README.md index 64d3930c106..c042da70deb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SelfHarmEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SelfHarmEvaluator.cs index 5946bbf0a7b..deb1d81834a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SelfHarmEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SelfHarmEvaluator.cs @@ -6,8 +6,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of content that indicates self harm. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of content that indicates self harm. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SexualEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SexualEvaluator.cs index bd5445ddd86..ed8093f5310 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SexualEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/SexualEvaluator.cs @@ -6,8 +6,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of sexual content. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of sexual content. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/UngroundedAttributesEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/UngroundedAttributesEvaluator.cs index a08acb45a0f..06019969345 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/UngroundedAttributesEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/UngroundedAttributesEvaluator.cs @@ -11,8 +11,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for presence of content that indicates ungrounded inference of human attributes. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for presence of content that indicates ungrounded inference of human attributes. /// /// /// @@ -31,7 +31,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// ignored. /// /// -/// The Azure AI Content Safety service uses a finetuned model to perform this evaluation which is expected to +/// The Azure AI Foundry Evaluation service uses a finetuned model to perform this evaluation which is expected to /// produce more accurate results than similar evaluations performed using a regular (non-finetuned) model. /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ViolenceEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ViolenceEvaluator.cs index 99928ff8184..3c7c5494fdf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ViolenceEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ViolenceEvaluator.cs @@ -6,8 +6,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// An that utilizes the Azure AI Content Safety service to evaluate responses produced by an -/// AI model for the presence of violent content. +/// An that utilizes the Azure AI Foundry Evaluation service to evaluate responses produced by +/// an AI model for the presence of violent content. /// /// /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs index 025ef58b809..1ba8ae270e6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs @@ -20,14 +20,14 @@ public enum EvaluationRating Inconclusive, /// - /// A value that indicates that the is interpreted as being exceptional. + /// A value that indicates that the is interpreted as being unacceptable. /// - Exceptional, + Unacceptable, /// - /// A value that indicates that the is interpreted as being good. + /// A value that indicates that the is interpreted as being poor. /// - Good, + Poor, /// /// A value that indicates that the is interpreted as being average. @@ -35,12 +35,12 @@ public enum EvaluationRating Average, /// - /// A value that indicates that the is interpreted as being poor. + /// A value that indicates that the is interpreted as being good. /// - Poor, + Good, /// - /// A value that indicates that the is interpreted as being unacceptable. + /// A value that indicates that the is interpreted as being exceptional. /// - Unacceptable, + Exceptional } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs index 3ffda8fb8f9..8d25085f4e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs @@ -131,17 +131,12 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -225,17 +220,13 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj index 703c83cb390..3f098cf3026 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/Microsoft.Extensions.AI.Evaluation.csproj @@ -1,7 +1,7 @@ - A library containing core abstractions for evaluating responses received from an LLM. + A library that defines core abstractions and types for supporting evaluation. $(TargetFrameworks);netstandard2.0 Microsoft.Extensions.AI.Evaluation diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/README.md b/src/Libraries/Microsoft.Extensions.AI.Evaluation/README.md index ba6f61c446a..c21e2a299ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/README.md +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/README.md @@ -4,7 +4,7 @@ * [`Microsoft.Extensions.AI.Evaluation`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation) - Defines core abstractions and types for supporting evaluation. * [`Microsoft.Extensions.AI.Evaluation.Quality`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Quality) - Contains evaluators that can be used to evaluate the quality of AI responses in your projects including Relevance, Truth, Completeness, Fluency, Coherence, Retrieval, Equivalence and Groundedness. -* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Content Safety service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. +* [`Microsoft.Extensions.AI.Evaluation.Safety`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Safety) - Contains a set of evaluators that are built atop the Azure AI Foundry Evaluation service that can be used to evaluate the content safety of AI responses in your projects including Protected Material, Groundedness Pro, Ungrounded Attributes, Hate and Unfairness, Self Harm, Violence, Sexual, Code Vulnerability and Indirect Attack. * [`Microsoft.Extensions.AI.Evaluation.Reporting`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting) - Contains support for caching LLM responses, storing the results of evaluations and generating reports from that data. * [`Microsoft.Extensions.AI.Evaluation.Reporting.Azure`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Reporting.Azure) - Supports the `Microsoft.Extensions.AI.Evaluation.Reporting` library with an implementation for caching LLM responses and storing the evaluation results in an Azure Storage container. * [`Microsoft.Extensions.AI.Evaluation.Console`](https://www.nuget.org/packages/Microsoft.Extensions.AI.Evaluation.Console) - A command line dotnet tool for generating reports and managing evaluation data. @@ -16,6 +16,7 @@ From the command-line: ```console dotnet add package Microsoft.Extensions.AI.Evaluation dotnet add package Microsoft.Extensions.AI.Evaluation.Quality +dotnet add package Microsoft.Extensions.AI.Evaluation.Safety dotnet add package Microsoft.Extensions.AI.Evaluation.Reporting ``` @@ -25,6 +26,7 @@ Or directly in the C# project file: + ``` diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md index 9cee8a46073..8822f8ddaea 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## 9.4.3-preview.1.25230.7 + +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.0-preview.1.25207.5 - Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. @@ -29,4 +33,4 @@ ## 9.0.0-preview.9.24507.7 -Initial Preview +- Initial Preview diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs index f42f1e1edfb..28f8eb8c3ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs @@ -25,6 +25,11 @@ public sealed class OllamaChatClient : IChatClient { private static readonly JsonElement _schemalessJsonResponseFormatValue = JsonDocument.Parse("\"json\"").RootElement; + private static readonly AIJsonSchemaTransformCache _schemaTransformCache = new(new() + { + ConvertBooleanSchemas = true, + }); + /// Metadata about the client. private readonly ChatClientMetadata _metadata; @@ -292,7 +297,7 @@ private static FunctionCallContent ToFunctionCallContent(OllamaFunctionToolCall { if (format is ChatResponseFormatJson jsonFormat) { - return jsonFormat.Schema ?? _schemalessJsonResponseFormatValue; + return _schemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) ?? _schemalessJsonResponseFormatValue; } else { @@ -402,12 +407,7 @@ private IEnumerable ToOllamaChatRequestMessages(ChatMe if (item is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { IList images = currentTextMessage?.Images ?? []; - images.Add(Convert.ToBase64String(dataContent.Data -#if NET - .Span)); -#else - .ToArray())); -#endif + images.Add(dataContent.Base64Data.ToString()); if (currentTextMessage is not null) { @@ -488,7 +488,7 @@ private static OllamaTool ToOllamaTool(AIFunction function) { Name = function.Name, Description = function.Description, - Parameters = JsonSerializer.Deserialize(function.JsonSchema, JsonContext.Default.OllamaFunctionToolParameters)!, + Parameters = JsonSerializer.Deserialize(_schemaTransformCache.GetOrCreateTransformedSchema(function), JsonContext.Default.OllamaFunctionToolParameters)!, } }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md index dc5dc62cc6c..05130ba3847 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 9.4.3-preview.1.25230.7 + +- Reverted previous change that enabled `strict` schemas by default. +- Updated `IChatClient` implementations to support `DataContent`s for PDFs. +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.0-preview.1.25207.5 - Updated to OpenAI 2.2.0-beta-4. @@ -41,4 +47,4 @@ ## 9.0.0-preview.9.24507.7 -Initial Preview +- Initial Preview diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index c78b495393b..c644bd77f21 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -18,12 +18,21 @@ #pragma warning disable EA0011 // Consider removing unnecessary conditional access operator (?) #pragma warning disable S1067 // Expressions should not be too complex #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields +#pragma warning disable SA1204 // Static elements should appear before instance elements namespace Microsoft.Extensions.AI; /// Represents an for an OpenAI or . internal sealed partial class OpenAIChatClient : IChatClient { + /// Gets the JSON schema transformer cache conforming to OpenAI restrictions per https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas. + internal static AIJsonSchemaTransformCache SchemaTransformCache { get; } = new(new() + { + RequireAllProperties = true, + DisallowAdditionalProperties = true, + ConvertBooleanSchemas = true + }); + /// Gets the default OpenAI endpoint. private static Uri DefaultOpenAIEndpoint { get; } = new("https://api.openai.com/v1"); @@ -155,21 +164,22 @@ void IDisposable.Dispose() foreach (var content in input.Contents) { - if (content is FunctionCallContent callRequest) + switch (content) { - message.ToolCalls.Add( - ChatToolCall.CreateFunctionToolCall( - callRequest.CallId, - callRequest.Name, - new(JsonSerializer.SerializeToUtf8Bytes( - callRequest.Arguments, - options.GetTypeInfo(typeof(IDictionary)))))); - } - } + case ErrorContent errorContent when errorContent.ErrorCode is nameof(message.Refusal): + message.Refusal = errorContent.Message; + break; - if (input.AdditionalProperties?.TryGetValue(nameof(message.Refusal), out string? refusal) is true) - { - message.Refusal = refusal; + case FunctionCallContent callRequest: + message.ToolCalls.Add( + ChatToolCall.CreateFunctionToolCall( + callRequest.CallId, + callRequest.Name, + new(JsonSerializer.SerializeToUtf8Bytes( + callRequest.Arguments, + options.GetTypeInfo(typeof(IDictionary)))))); + break; + } } yield return message; @@ -250,7 +260,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha string? responseId = null; DateTimeOffset? createdAt = null; string? modelId = null; - string? fingerprint = null; // Process each update as it arrives await foreach (StreamingChatCompletionUpdate update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -261,7 +270,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha responseId ??= update.CompletionId; createdAt ??= update.CreatedAt; modelId ??= update.Model; - fingerprint ??= update.SystemFingerprint; // Create the response content object. ChatResponseUpdate responseUpdate = new() @@ -275,22 +283,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha Role = streamedRole, }; - // Populate it with any additional metadata from the OpenAI object. - if (update.ContentTokenLogProbabilities is { Count: > 0 } contentTokenLogProbs) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.ContentTokenLogProbabilities)] = contentTokenLogProbs; - } - - if (update.RefusalTokenLogProbabilities is { Count: > 0 } refusalTokenLogProbs) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.RefusalTokenLogProbabilities)] = refusalTokenLogProbs; - } - - if (fingerprint is not null) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.SystemFingerprint)] = fingerprint; - } - // Transfer over content update items. if (update.ContentUpdate is { Count: > 0 }) { @@ -370,13 +362,7 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha // add it to this function calling item. if (refusal is not null) { - (responseUpdate.AdditionalProperties ??= [])[nameof(ChatMessageContentPart.Refusal)] = refusal.ToString(); - } - - // Propagate additional relevant metadata. - if (fingerprint is not null) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(ChatCompletion.SystemFingerprint)] = fingerprint; + responseUpdate.Contents.Add(new ErrorContent(refusal.ToString()) { ErrorCode = "Refusal" }); } yield return responseUpdate; @@ -417,20 +403,7 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple "mp3" or _ => "audio/mpeg", }; - var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType) - { - AdditionalProperties = new() { [nameof(audio.ExpiresAt)] = audio.ExpiresAt }, - }; - - if (audio.Id is string id) - { - dc.AdditionalProperties[nameof(audio.Id)] = id; - } - - if (audio.Transcript is string transcript) - { - dc.AdditionalProperties[nameof(audio.Transcript)] = transcript; - } + var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType); returnMessage.Contents.Add(dc); } @@ -450,6 +423,12 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple } } + // And add error content for any refusals, which represent errors in generating output that conforms to a provided schema. + if (openAICompletion.Refusal is string refusal) + { + returnMessage.Contents.Add(new ErrorContent(refusal) { ErrorCode = nameof(openAICompletion.Refusal) }); + } + // Wrap the content in a ChatResponse to return. var response = new ChatResponse(returnMessage) { @@ -465,152 +444,81 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple response.Usage = FromOpenAIUsage(tokenUsage); } - if (openAICompletion.ContentTokenLogProbabilities is { Count: > 0 } contentTokenLogProbs) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.ContentTokenLogProbabilities)] = contentTokenLogProbs; - } - - if (openAICompletion.Refusal is string refusal) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.Refusal)] = refusal; - } - - if (openAICompletion.RefusalTokenLogProbabilities is { Count: > 0 } refusalTokenLogProbs) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.RefusalTokenLogProbabilities)] = refusalTokenLogProbs; - } - - if (openAICompletion.SystemFingerprint is string systemFingerprint) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.SystemFingerprint)] = systemFingerprint; - } - return response; } /// Converts an extensions options instance to an OpenAI options instance. - private static ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) + private ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) { - ChatCompletionOptions result = new(); + if (options is null) + { + return new ChatCompletionOptions(); + } - if (options is not null) + if (options.RawRepresentationFactory?.Invoke(this) is not ChatCompletionOptions result) { - result.FrequencyPenalty = options.FrequencyPenalty; - result.MaxOutputTokenCount = options.MaxOutputTokens; - result.TopP = options.TopP; - result.PresencePenalty = options.PresencePenalty; - result.Temperature = options.Temperature; - result.AllowParallelToolCalls = options.AllowMultipleToolCalls; + result = new ChatCompletionOptions(); + } + + result.FrequencyPenalty ??= options.FrequencyPenalty; + result.MaxOutputTokenCount ??= options.MaxOutputTokens; + result.TopP ??= options.TopP; + result.PresencePenalty ??= options.PresencePenalty; + result.Temperature ??= options.Temperature; + result.AllowParallelToolCalls ??= options.AllowMultipleToolCalls; #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. - result.Seed = options.Seed; + result.Seed ??= options.Seed; #pragma warning restore OPENAI001 - if (options.StopSequences is { Count: > 0 } stopSequences) + if (options.StopSequences is { Count: > 0 } stopSequences) + { + foreach (string stopSequence in stopSequences) { - foreach (string stopSequence in stopSequences) - { - result.StopSequences.Add(stopSequence); - } + result.StopSequences.Add(stopSequence); } + } - if (options.AdditionalProperties is { Count: > 0 } additionalProperties) + if (options.Tools is { Count: > 0 } tools) + { + foreach (AITool tool in tools) { - if (additionalProperties.TryGetValue(nameof(result.AudioOptions), out ChatAudioOptions? audioOptions)) - { - result.AudioOptions = audioOptions; - } - - if (additionalProperties.TryGetValue(nameof(result.EndUserId), out string? endUserId)) - { - result.EndUserId = endUserId; - } - - if (additionalProperties.TryGetValue(nameof(result.IncludeLogProbabilities), out bool includeLogProbabilities)) - { - result.IncludeLogProbabilities = includeLogProbabilities; - } - - if (additionalProperties.TryGetValue(nameof(result.LogitBiases), out IDictionary? logitBiases)) - { - foreach (KeyValuePair kvp in logitBiases!) - { - result.LogitBiases[kvp.Key] = kvp.Value; - } - } - - if (additionalProperties.TryGetValue(nameof(result.Metadata), out IDictionary? metadata)) - { - foreach (KeyValuePair kvp in metadata) - { - result.Metadata[kvp.Key] = kvp.Value; - } - } - - if (additionalProperties.TryGetValue(nameof(result.OutputPrediction), out ChatOutputPrediction? outputPrediction)) - { - result.OutputPrediction = outputPrediction; - } - - if (additionalProperties.TryGetValue(nameof(result.ReasoningEffortLevel), out ChatReasoningEffortLevel reasoningEffortLevel)) - { - result.ReasoningEffortLevel = reasoningEffortLevel; - } - - if (additionalProperties.TryGetValue(nameof(result.ResponseModalities), out ChatResponseModalities responseModalities)) - { - result.ResponseModalities = responseModalities; - } - - if (additionalProperties.TryGetValue(nameof(result.StoredOutputEnabled), out bool storeOutputEnabled)) + if (tool is AIFunction af) { - result.StoredOutputEnabled = storeOutputEnabled; - } - - if (additionalProperties.TryGetValue(nameof(result.TopLogProbabilityCount), out int topLogProbabilityCountInt)) - { - result.TopLogProbabilityCount = topLogProbabilityCountInt; + result.Tools.Add(ToOpenAIChatTool(af)); } } - if (options.Tools is { Count: > 0 } tools) + if (result.ToolChoice is null && result.Tools.Count > 0) { - foreach (AITool tool in tools) + switch (options.ToolMode) { - if (tool is AIFunction af) - { - result.Tools.Add(ToOpenAIChatTool(af)); - } - } - - if (result.Tools.Count > 0) - { - switch (options.ToolMode) - { - case NoneChatToolMode: - result.ToolChoice = ChatToolChoice.CreateNoneChoice(); - break; - - case AutoChatToolMode: - case null: - result.ToolChoice = ChatToolChoice.CreateAutoChoice(); - break; - - case RequiredChatToolMode required: - result.ToolChoice = required.RequiredFunctionName is null ? - ChatToolChoice.CreateRequiredChoice() : - ChatToolChoice.CreateFunctionChoice(required.RequiredFunctionName); - break; - } + case NoneChatToolMode: + result.ToolChoice = ChatToolChoice.CreateNoneChoice(); + break; + + case AutoChatToolMode: + case null: + result.ToolChoice = ChatToolChoice.CreateAutoChoice(); + break; + + case RequiredChatToolMode required: + result.ToolChoice = required.RequiredFunctionName is null ? + ChatToolChoice.CreateRequiredChoice() : + ChatToolChoice.CreateFunctionChoice(required.RequiredFunctionName); + break; } } + } + if (result.ResponseFormat is null) + { if (options.ResponseFormat is ChatResponseFormatText) { result.ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat(); } else if (options.ResponseFormat is ChatResponseFormatJson jsonFormat) { - result.ResponseFormat = jsonFormat.Schema is { } jsonSchema ? + result.ResponseFormat = SchemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) is { } jsonSchema ? OpenAI.Chat.ChatResponseFormat.CreateJsonSchemaFormat( jsonFormat.SchemaName ?? "json_schema", BinaryData.FromBytes( @@ -631,8 +539,11 @@ private static ChatTool ToOpenAIChatTool(AIFunction aiFunction) strictObj is bool strictValue ? strictValue : null; + // Perform transformations making the schema legal per OpenAI restrictions + JsonElement jsonSchema = SchemaTransformCache.GetOrCreateTransformedSchema(aiFunction); + // Map to an intermediate model so that redundant properties are skipped. - var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema, ChatClientJsonContext.Default.ChatToolJson)!; + var tool = JsonSerializer.Deserialize(jsonSchema, ChatClientJsonContext.Default.ChatToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool, ChatClientJsonContext.Default.ChatToolJson)); return ChatTool.CreateFunctionTool(aiFunction.Name, aiFunction.Description, functionParameters, strict); } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs index 92224379c10..a91ea9abf8f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs @@ -263,6 +263,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( MessageId = lastMessageId, ModelId = modelId, ResponseId = responseId, + Role = lastRole, ConversationId = responseId, Contents = [ @@ -274,6 +275,19 @@ public async IAsyncEnumerable GetStreamingResponseAsync( ], }; break; + + case StreamingResponseRefusalDoneUpdate refusalDone: + yield return new ChatResponseUpdate + { + CreatedAt = createdAt, + MessageId = lastMessageId, + ModelId = modelId, + ResponseId = responseId, + Role = lastRole, + ConversationId = responseId, + Contents = [new ErrorContent(refusalDone.Refusal) { ErrorCode = nameof(ResponseContentPart.Refusal) }], + }; + break; } } } @@ -359,7 +373,7 @@ private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptio switch (tool) { case AIFunction af: - var oaitool = JsonSerializer.Deserialize(af.JsonSchema, ResponseClientJsonContext.Default.ResponseToolJson)!; + var oaitool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(af), ResponseClientJsonContext.Default.ResponseToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(oaitool, ResponseClientJsonContext.Default.ResponseToolJson)); result.Tools.Add(ResponseTool.CreateFunctionTool(af.Name, af.Description, functionParameters)); break; @@ -414,7 +428,7 @@ private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptio { result.TextOptions = new() { - TextFormat = jsonFormat.Schema is { } jsonSchema ? + TextFormat = SchemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) is { } jsonSchema ? ResponseTextFormat.CreateJsonSchemaFormat( jsonFormat.SchemaName ?? "json_schema", BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(jsonSchema, ResponseClientJsonContext.Default.JsonElement)), @@ -539,9 +553,15 @@ private static List ToAIContents(IEnumerable con foreach (ResponseContentPart part in contents) { - if (part.Kind == ResponseContentPartKind.OutputText) + switch (part.Kind) { - results.Add(new TextContent(part.Text)); + case ResponseContentPartKind.OutputText: + results.Add(new TextContent(part.Text)); + break; + + case ResponseContentPartKind.Refusal: + results.Add(new ErrorContent(part.Refusal) { ErrorCode = nameof(ResponseContentPartKind.Refusal) }); + break; } } @@ -572,6 +592,10 @@ private static List ToOpenAIResponsesContent(IList` arguments. +- Unsealed `FunctionInvocationContext`. +- Added `FunctionInvocationContext.IsStreaming`. +- Added protected `FunctionInvokingChatClient.FunctionInvocationServices` property to surface the corresponding `IServiceProvider` provided at construction time. +- Changed protected virtual `FunctionInvokingChatClient.InvokeFunctionAsync` to return `ValueTask` instead of `Task`. Diagnostics are now emitted even if the method is overridden. +- Added `FunctionInvocationResult.Terminate`. + ## 9.4.0-preview.1.25207.5 - Updated `GetResponseAsync` to default to using JSON-schema based structured output by default. @@ -74,4 +89,4 @@ ## 9.0.0-preview.9.24507.7 -Initial Preview +- Initial Preview diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs index 5aa70e4b262..211fc39ec85 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs @@ -9,6 +9,7 @@ using Microsoft.Shared.Diagnostics; #pragma warning disable S127 // "for" loop stop conditions should be invariant +#pragma warning disable SA1202 // Elements should be ordered by access namespace Microsoft.Extensions.AI; @@ -45,11 +46,19 @@ protected CachingChatClient(IChatClient innerClient) public bool CoalesceStreamingUpdates { get; set; } = true; /// - public override async Task GetResponseAsync( + public override Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(messages); + return UseCaching(options) ? + GetCachedResponseAsync(messages, options, cancellationToken) : + base.GetResponseAsync(messages, options, cancellationToken); + } + + private async Task GetCachedResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { // We're only storing the final result, not the in-flight task, so that we can avoid caching failures // or having problems when one of the callers cancels but others don't. This has the drawback that // concurrent callers might trigger duplicate requests, but that's acceptable. @@ -65,11 +74,19 @@ public override async Task GetResponseAsync( } /// - public override async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public override IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(messages); + return UseCaching(options) ? + GetCachedStreamingResponseAsync(messages, options, cancellationToken) : + base.GetStreamingResponseAsync(messages, options, cancellationToken); + } + + private async IAsyncEnumerable GetCachedStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { if (CoalesceStreamingUpdates) { // When coalescing updates, we cache non-streaming results coalesced from streaming ones. That means @@ -178,4 +195,13 @@ public override async IAsyncEnumerable GetStreamingResponseA /// is . /// is . protected abstract Task WriteCacheStreamingAsync(string key, IReadOnlyList value, CancellationToken cancellationToken); + + /// Determine whether to use caching with the request. + private static bool UseCaching(ChatOptions? options) + { + // We want to skip caching if options.ConversationId is set. If it's set, that implies there's + // some state that will impact the response and that's not represented in the messages. Since + // that state could change even with the same ID, we have to assume caching isn't valid. + return options?.ConversationId is null; + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientBuilder.cs index bae7d58e4f9..c557ba524ff 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientBuilder.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientBuilder.cs @@ -66,6 +66,7 @@ public IChatClient Build(IServiceProvider? services = null) /// The client factory function. /// The updated instance. /// is . + /// Pipelines of functionality. public ChatClientBuilder Use(Func clientFactory) { _ = Throw.IfNull(clientFactory); @@ -77,6 +78,7 @@ public ChatClientBuilder Use(Func clientFactory) /// The client factory function. /// The updated instance. /// is . + /// Pipelines of functionality. public ChatClientBuilder Use(Func clientFactory) { _ = Throw.IfNull(clientFactory); @@ -98,10 +100,11 @@ public ChatClientBuilder Use(Func cl /// /// The updated instance. /// - /// This overload may be used when the anonymous implementation needs to provide pre- and/or post-processing, but doesn't + /// This overload can be used when the anonymous implementation needs to provide pre-processing and/or post-processing, but doesn't /// need to interact with the results of the operation, which will come from the inner client. /// /// is . + /// Pipelines of functionality. public ChatClientBuilder Use(Func, ChatOptions?, Func, ChatOptions?, CancellationToken, Task>, CancellationToken, Task> sharedFunc) { _ = Throw.IfNull(sharedFunc); @@ -125,7 +128,7 @@ public ChatClientBuilder Use(Func, ChatOptions?, Func /// The updated instance. /// - /// One or both delegates may be provided. If both are provided, they will be used for their respective methods: + /// One or both delegates can be provided. If both are provided, they will be used for their respective methods: /// will provide the implementation of , and /// will provide the implementation of . /// If only one of the delegates is provided, it will be used for both methods. That means that if @@ -135,6 +138,7 @@ public ChatClientBuilder Use(Func, ChatOptions?, Func will be implemented by combining the updates from . /// /// Both and are . + /// Pipelines of functionality. public ChatClientBuilder Use( Func, ChatOptions?, IChatClient, CancellationToken, Task>? getResponseFunc, Func, ChatOptions?, IChatClient, CancellationToken, IAsyncEnumerable>? getStreamingResponseFunc) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs index 90fe7ac1157..d7bc12a1a41 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs @@ -18,6 +18,7 @@ namespace Microsoft.Extensions.AI; /// /// Provides extension methods on that simplify working with structured output. /// +/// Request a response with structured output. public static class ChatClientStructuredOutputExtensions { private static readonly AIJsonSchemaCreateOptions _inferenceOptions = new() @@ -32,10 +33,9 @@ public static class ChatClientStructuredOutputExtensions /// The . /// The chat content to send. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is . + /// + /// to set a JSON schema on the ; otherwise, . The default is . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. @@ -44,39 +44,37 @@ public static Task> GetResponseAsync( this IChatClient chatClient, IEnumerable messages, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, messages, AIJsonUtilities.DefaultOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, messages, AIJsonUtilities.DefaultOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a user chat text message, requesting a response matching the type . /// The . /// The text content for the chat message to send. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is determined by the implementation. - /// If a specific value is required, it must be specified by the caller. + /// + /// to set a JSON schema on the ; otherwise, . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. /// The type of structured output to request. + /// Request a response with structured output. public static Task> GetResponseAsync( this IChatClient chatClient, string chatMessage, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a chat message, requesting a response matching the type . /// The . /// The chat message to send. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is . + /// + /// to set a JSON schema on the ; otherwise, . The default is . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. @@ -85,19 +83,18 @@ public static Task> GetResponseAsync( this IChatClient chatClient, ChatMessage chatMessage, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, [chatMessage], options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, [chatMessage], options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a user chat text message, requesting a response matching the type . /// The . /// The text content for the chat message to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is . + /// + /// to set a JSON schema on the ; otherwise, . The default is . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. @@ -107,19 +104,18 @@ public static Task> GetResponseAsync( string chatMessage, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), serializerOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), serializerOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a chat message, requesting a response matching the type . /// The . /// The chat message to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is . + /// + /// to set a JSON schema on the ; otherwise, . The default is . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. @@ -129,32 +125,29 @@ public static Task> GetResponseAsync( ChatMessage chatMessage, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, [chatMessage], serializerOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, [chatMessage], serializerOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends chat messages, requesting a response matching the type . /// The . /// The chat content to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// - /// Optionally specifies whether to set a JSON schema on the . - /// This improves reliability if the underlying model supports native structured output with a schema, but may cause an error if the model does not support it. - /// If not specified, the default value is . + /// + /// to set a JSON schema on the ; otherwise, . The default is . + /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. /// The type of structured output to request. - /// is . - /// is . - /// is . + /// or or is . public static async Task> GetResponseAsync( this IChatClient chatClient, IEnumerable messages, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(chatClient); @@ -197,8 +190,8 @@ public static async Task> GetResponseAsync( // We default to assuming that models support JSON schema because developers will normally use // GetResponseAsync only with models that do. If the model doesn't support JSON schema, it may - // throw or it may ignore the schema. In these cases developers should pass useJsonSchema: false. - if (useJsonSchema.GetValueOrDefault(true)) + // throw or it may ignore the schema. In these cases developers should pass useJsonSchemaResponseFormat: false. + if (useJsonSchemaResponseFormat ?? true) { // When using native structured output, we don't add any additional prompt, because // the LLM backend is meant to do whatever's needed to explain the schema to the LLM. diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ConfigureOptionsChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ConfigureOptionsChatClientBuilderExtensions.cs index d76b2ba1a2e..5f9c2f988c0 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ConfigureOptionsChatClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ConfigureOptionsChatClientBuilderExtensions.cs @@ -27,6 +27,7 @@ public static class ConfigureOptionsChatClientBuilderExtensions /// The . /// is . /// is . + /// Provide options. public static ChatClientBuilder ConfigureOptions( this ChatClientBuilder builder, Action configure) { diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index bf016dfc2e4..d624ec63abb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -15,7 +15,6 @@ using Microsoft.Shared.Diagnostics; #pragma warning disable CA2213 // Disposable fields should be disposed -#pragma warning disable CA1002 // Do not expose generic lists #pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test #pragma warning disable SA1202 // 'protected' members should come before 'private' members #pragma warning disable S107 // Methods should not have too many parameters @@ -214,7 +213,7 @@ public override async Task GetResponseAsync( // A single request into this GetResponseAsync may result in multiple requests to the inner client. // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient)); + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetResponseAsync)}"); // Copy the original messages in order to avoid enumerating the original messages multiple times. // The IEnumerable can represent an arbitrary amount of work. @@ -309,7 +308,7 @@ public override async IAsyncEnumerable GetStreamingResponseA // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client. // Create an activity to group them together for better observability. - using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient)); + using Activity? activity = _activitySource?.StartActivity($"{nameof(FunctionInvokingChatClient)}.{nameof(GetStreamingResponseAsync)}"); UsageDetails? totalUsage = activity is { IsAllDataRequested: true } ? new() : null; // tracked usage across all turns, to be used for activity purposes // Copy the original messages in order to avoid enumerating the original messages multiple times. @@ -795,7 +794,16 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul { _ = Throw.IfNull(context); - using Activity? activity = _activitySource?.StartActivity(context.Function.Name); + using Activity? activity = _activitySource?.StartActivity( + $"execute_tool {context.Function.Name}", + ActivityKind.Internal, + default(ActivityContext), + [ + new(OpenTelemetryConsts.GenAI.Operation.Name, "execute_tool"), + new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId), + new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name), + new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description), + ]); long startingTimestamp = 0; if (_logger.IsEnabled(LogLevel.Debug)) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs index b5f43f5385b..3937d5db59b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs @@ -13,10 +13,18 @@ namespace Microsoft.Extensions.AI; /// A delegating chat client that logs chat operations to an . +/// /// /// The provided implementation of is thread-safe for concurrent use so long as the /// employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// chat messages and options are logged. These messages and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// public partial class LoggingChatClient : DelegatingChatClient { /// An instance used for all logging. diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs index d34716ed886..e2759b6b0a6 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs @@ -21,6 +21,14 @@ public static class LoggingChatClientBuilderExtensions /// An optional callback that can be used to configure the instance. /// The . /// is . + /// + /// + /// When the employed enables , the contents of + /// chat messages and options are logged. These messages and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static ChatClientBuilder UseLogging( this ChatClientBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index c74bd3aa3c1..c22dc292c8a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -25,7 +25,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// public sealed partial class OpenTelemetryChatClient : DelegatingChatClient diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs index 924ee362633..97e5beb2c42 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs @@ -14,10 +14,18 @@ namespace Microsoft.Extensions.AI; /// A delegating embedding generator that logs embedding generation operations to an . /// Specifies the type of the input passed to the generator. /// Specifies the type of the embedding instance produced by the generator. +/// /// /// The provided implementation of is thread-safe for concurrent use /// so long as the employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// values and options are logged. These values and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// public partial class LoggingEmbeddingGenerator : DelegatingEmbeddingGenerator where TEmbedding : Embedding { diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs index eb472fb1e0e..a7afbdeed85 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs @@ -23,6 +23,14 @@ public static class LoggingEmbeddingGeneratorBuilderExtensions /// An optional callback that can be used to configure the instance. /// The . /// is . + /// + /// + /// When the employed enables , the contents of + /// values and options are logged. These values and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static EmbeddingGeneratorBuilder UseLogging( this EmbeddingGeneratorBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 14332d1253f..99a3ed684af 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -19,7 +19,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.32, defined at . +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.33, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The type of input used to produce embeddings. @@ -86,6 +86,20 @@ public OpenTelemetryEmbeddingGenerator(IEmbeddingGenerator i ); } + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is . + /// + /// + /// By default, telemetry includes metadata, such as token counts, but not raw inputs + /// and outputs or additional options data. + /// + public bool EnableSensitiveData { get; set; } + /// public override object? GetService(Type serviceType, object? serviceKey = null) => serviceType == typeof(ActivitySource) ? _activitySource : @@ -163,20 +177,19 @@ protected override void Dispose(bool disposing) _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensionsValue); } - if (options is not null && - _system is not null) + // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where + // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), + // and more generally cases where there's additional useful information to be logged. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + if (EnableSensitiveData && + _system is not null && + options?.AdditionalProperties is { } props) { - // Log all additional request options as per-provider tags. This is non-normative, but it covers cases where - // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.request.service_tier), - // and more generally cases where there's additional useful information to be logged. - if (options.AdditionalProperties is { } props) + foreach (KeyValuePair prop in props) { - foreach (KeyValuePair prop in props) - { - _ = activity.AddTag( - OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), - prop.Value); - } + _ = activity.AddTag( + OpenTelemetryConsts.GenAI.Request.PerProvider(_system, JsonNamingPolicy.SnakeCaseLower.ConvertName(prop.Key)), + prop.Value); } } } @@ -247,7 +260,8 @@ private void TraceResponse( // Log all additional response properties as per-provider tags. This is non-normative, but it covers cases where // there's a per-provider specification in a best-effort manner (e.g. gen_ai.openai.response.system_fingerprint), // and more generally cases where there's additional useful information to be logged. - if (_system is not null && + if (EnableSensitiveData && + _system is not null && embeddings?.AdditionalProperties is { } props) { foreach (KeyValuePair prop in props) diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index b585ebf3086..c1a22066227 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -103,7 +103,14 @@ public static class Token public static class Tool { + public const string Name = "gen_ai.tool.name"; + public const string Description = "gen_ai.tool.description"; public const string Message = "gen_ai.tool.message"; + + public static class Call + { + public const string Id = "gen_ai.tool.call.id"; + } } public static class User diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs index 6c5bf0ed929..e7bf7850a94 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs @@ -15,10 +15,18 @@ namespace Microsoft.Extensions.AI; /// A delegating speech to text client that logs speech to text operations to an . +/// /// /// The provided implementation of is thread-safe for concurrent use so long as the /// employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// messages and options are logged. These messages and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// [Experimental("MEAI001")] public partial class LoggingSpeechToTextClient : DelegatingSpeechToTextClient { diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs similarity index 79% rename from src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs index 7ce2b19ac37..92a67189982 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs @@ -12,7 +12,7 @@ namespace Microsoft.Extensions.AI; /// Provides extensions for configuring instances. [Experimental("MEAI001")] -public static class SpeechToTextClientBuilderExtensions +public static class LoggingSpeechToTextClientBuilderExtensions { /// Adds logging to the audio transcription client pipeline. /// The . @@ -22,6 +22,14 @@ public static class SpeechToTextClientBuilderExtensions /// /// An optional callback that can be used to configure the instance. /// The . + /// + /// + /// When the employed enables , the contents of + /// messages and options are logged. These messages and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static SpeechToTextClientBuilder UseLogging( this SpeechToTextClientBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md index 0f0238ec1ca..37f10b83ce2 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md @@ -240,6 +240,10 @@ Make sure to replace `YOUR-AZURE-AI-SEARCH-KEY` and `YOUR-AZURE-AI-SEARCH-ENDPOI 3. Once installed, Open the `Program.cs` file. 4. Run the project by clicking the "Run" button in the Debug view. +# Updating JavaScript dependencies + +This template leverages JavaScript libraries to provide essential functionality. These libraries are located in the wwwroot/lib folder. For instructions on updating each dependency, please refer to the README.md file in each respective folder. + # Learn More To learn more about development with .NET and AI, check out the following links: diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/dompurify/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/dompurify/README.md index 8ccedde96cb..5d34e078c3a 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/dompurify/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/dompurify/README.md @@ -2,4 +2,4 @@ dompurify version 3.2.4 https://github.com/cure53/DOMPurify License: Apache 2.0 and Mozilla Public License 2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/dompurify +To update, replace the `dist/purify.es.mjs` file with an updated version from https://www.npmjs.com/package/dompurify. diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/marked/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/marked/README.md index 31aeeb10a82..9f34411f17a 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/marked/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/marked/README.md @@ -2,4 +2,4 @@ marked version 15.0.6 https://github.com/markedjs/marked License: MIT -To update, replace the files with with an updated build from https://www.npmjs.com/package/marked +To update, replace the `dist/marked.esm.js` file with with an updated version from https://www.npmjs.com/package/marked. diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/pdfjs-dist/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/pdfjs-dist/README.md index 1ec731136cb..9c5ca581563 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/pdfjs-dist/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/wwwroot/lib/pdfjs-dist/README.md @@ -2,4 +2,9 @@ pdfjs-dist version 4.10.38 https://github.com/mozilla/pdf.js License: Apache-2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/pdfjs-dist +To update, replace the following files with updated versions from https://www.npmjs.com/package/pdfjs-dist: +* `build/pdf.min.mjs` +* `build/pdf.worker.min.mjs` +* `web/pdf_viewer.css` +* `web/pdf_viewer.mjs` +* `web/images/loading-icon.gif` diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md index 78b6b6f587d..f7c944dacc8 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md @@ -181,6 +181,10 @@ Several .NET Aspire templates include ASP.NET Core projects that are configured See [Troubleshoot untrusted localhost certificate in .NET Aspire](https://learn.microsoft.com/dotnet/aspire/troubleshooting/untrusted-localhost-certificate) for more information. +# Updating JavaScript dependencies + +This template leverages JavaScript libraries to provide essential functionality. These libraries are located in the wwwroot/lib folder of the ChatWithCustomData-CSharp.Web project. For instructions on updating each dependency, please refer to the README.md file in each respective folder. + # Learn More To learn more about development with .NET and AI, check out the following links: diff --git a/src/ProjectTemplates/README.md b/src/ProjectTemplates/README.md index 6f69a23450d..5ac7bb096f8 100644 --- a/src/ProjectTemplates/README.md +++ b/src/ProjectTemplates/README.md @@ -11,6 +11,16 @@ To update project template JavaScript dependencies: To add a new dependency, run `npm install ` and update the `scripts` section in `package.json` to specify how the new dependency should be copied into its template. +# Component governance + +There are two types of template dependencies that need to get scanned for component governance (CG): +* .NET dependencies (specified via `` in each `.csproj` file) +* JS dependencies (everything in the `wwwroot/lib` folder of the `.Web` project) + +There are template execution tests in the `test/ProjectTemplates` folder of this repo that create, restore, and build each possible variation of the template. These tests execute before the CG step of the internal CI pipeline, which scans the build artifacts from each generated project (namely the `project.assets.json` file and the local NuGet package cache) to detect which .NET dependencies got pulled in. + +However, CG can't detect JS dependencies by scanning execution test output, because the generated projects don't contain manifests describing JS dependencies. Instead, we have a `package.json` and `package-lock.json` in the same folder as this README that define which JS dependencies get included in the template and how they get copied into template content (see previous section in this document). CG then automatically tracks packages listed in this `package-lock.json`. + # Running AI templates ## Build the templates using just-built library package versions diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs index 67bbfb6d3db..cdf1aab09c9 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Text.Json; using Xunit; @@ -28,6 +29,7 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.ToolMode); Assert.Null(options.Tools); Assert.Null(options.AdditionalProperties); + Assert.Null(options.RawRepresentationFactory); ChatOptions clone = options.Clone(); Assert.Null(clone.ConversationId); @@ -45,6 +47,7 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(clone.ToolMode); Assert.Null(clone.Tools); Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.RawRepresentationFactory); } [Fact] @@ -69,6 +72,8 @@ public void Properties_Roundtrip() ["key"] = "value", }; + Func rawRepresentationFactory = (c) => null; + options.ConversationId = "12345"; options.Temperature = 0.1f; options.MaxOutputTokens = 2; @@ -83,6 +88,7 @@ public void Properties_Roundtrip() options.AllowMultipleToolCalls = true; options.ToolMode = ChatToolMode.RequireAny; options.Tools = tools; + options.RawRepresentationFactory = rawRepresentationFactory; options.AdditionalProperties = additionalProps; Assert.Equal("12345", options.ConversationId); @@ -99,6 +105,7 @@ public void Properties_Roundtrip() Assert.True(options.AllowMultipleToolCalls); Assert.Same(ChatToolMode.RequireAny, options.ToolMode); Assert.Same(tools, options.Tools); + Assert.Same(rawRepresentationFactory, options.RawRepresentationFactory); Assert.Same(additionalProps, options.AdditionalProperties); ChatOptions clone = options.Clone(); @@ -116,6 +123,7 @@ public void Properties_Roundtrip() Assert.True(clone.AllowMultipleToolCalls); Assert.Same(ChatToolMode.RequireAny, clone.ToolMode); Assert.Equal(tools, clone.Tools); + Assert.Same(rawRepresentationFactory, clone.RawRepresentationFactory); Assert.Equal(additionalProps, clone.AdditionalProperties); } @@ -153,6 +161,7 @@ public void JsonSerialization_Roundtrips() AIFunctionFactory.Create(() => 42), AIFunctionFactory.Create(() => 43), ]; + options.RawRepresentationFactory = (c) => null; options.AdditionalProperties = additionalProps; string json = JsonSerializer.Serialize(options, TestJsonSerializerContext.Default.ChatOptions); @@ -175,6 +184,7 @@ public void JsonSerialization_Roundtrips() Assert.False(deserialized.AllowMultipleToolCalls); Assert.Equal(ChatToolMode.RequireAny, deserialized.ToolMode); Assert.Null(deserialized.Tools); + Assert.Null(deserialized.RawRepresentationFactory); Assert.NotNull(deserialized.AdditionalProperties); Assert.Single(deserialized.AdditionalProperties); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs index 83f09c66889..0f5b6b22d92 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/DataContentTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Text; using System.Text.Json; using Xunit; @@ -66,21 +67,27 @@ public void Ctor_ValidMediaType_Roundtrips(string mediaType) { var content = new DataContent("data:image/png;base64,aGVsbG8=", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); content = new DataContent("data:,", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent("data:text/plain,", mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent(new Uri("data:text/plain,"), mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("", content.Base64Data.ToString()); content = new DataContent(new byte[] { 0, 1, 2 }, mediaType); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("AAEC", content.Base64Data.ToString()); content = new DataContent(content.Uri); Assert.Equal(mediaType, content.MediaType); + Assert.Equal("AAEC", content.Base64Data.ToString()); } [Fact] @@ -91,10 +98,12 @@ public void Ctor_NoMediaType_Roundtrips() content = new DataContent("data:image/png;base64,aGVsbG8="); Assert.Equal("data:image/png;base64,aGVsbG8=", content.Uri); Assert.Equal("image/png", content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); content = new DataContent(new Uri("data:image/png;base64,aGVsbG8=")); Assert.Equal("data:image/png;base64,aGVsbG8=", content.Uri); Assert.Equal("image/png", content.MediaType); + Assert.Equal("aGVsbG8=", content.Base64Data.ToString()); } [Fact] @@ -128,6 +137,7 @@ public void Deserialize_MatchesExpectedData() Assert.Equal("data:application/octet-stream;base64,AQIDBA==", content.Uri); Assert.Equal([0x01, 0x02, 0x03, 0x04], content.Data.ToArray()); + Assert.Equal("AQIDBA==", content.Base64Data.ToString()); Assert.Equal("application/octet-stream", content.MediaType); // Uri referenced content-only @@ -150,6 +160,7 @@ public void Deserialize_MatchesExpectedData() Assert.Equal("data:audio/wav;base64,AQIDBA==", content.Uri); Assert.Equal([0x01, 0x02, 0x03, 0x04], content.Data.ToArray()); + Assert.Equal("AQIDBA==", content.Base64Data.ToString()); Assert.Equal("audio/wav", content.MediaType); Assert.Equal("value", content.AdditionalProperties!["key"]!.ToString()); } @@ -224,4 +235,29 @@ public void HasMediaTypePrefix_ReturnsFalse(string mediaType, string prefix) var content = new DataContent("data:application/octet-stream;base64,AQIDBA==", mediaType); Assert.False(content.HasTopLevelMediaType(prefix)); } + + [Fact] + public void Data_Roundtrips() + { + Random rand = new(42); + for (int length = 0; length < 100; length++) + { + byte[] data = new byte[length]; + rand.NextBytes(data); + + var content = new DataContent(data, "application/octet-stream"); + Assert.Equal(data, content.Data.ToArray()); + Assert.Equal(Convert.ToBase64String(data), content.Base64Data.ToString()); + Assert.Equal($"data:application/octet-stream;base64,{Convert.ToBase64String(data)}", content.Uri); + } + } + + [Fact] + public void NonBase64Data_Normalized() + { + var content = new DataContent("data:text/plain,hello world"); + Assert.Equal("data:text/plain;base64,aGVsbG8gd29ybGQ=", content.Uri); + Assert.Equal("aGVsbG8gd29ybGQ=", content.Base64Data.ToString()); + Assert.Equal("hello world", Encoding.ASCII.GetString(content.Data.ToArray())); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs new file mode 100644 index 00000000000..c75d715466e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Linq; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class BinaryEmbeddingTests +{ + [Fact] + public void Ctor_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + Assert.Same(vector, e.Vector); + Assert.Null(e.ModelId); + Assert.Null(e.CreatedAt); + Assert.Null(e.AdditionalProperties); + } + + [Fact] + public void Properties_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + + Assert.Same(vector, e.Vector); + BitArray newVector = new BitArray(new bool[] { true, false, true, false }); + e.Vector = newVector; + Assert.Same(newVector, e.Vector); + + Assert.Null(e.ModelId); + e.ModelId = "text-embedding-3-small"; + Assert.Equal("text-embedding-3-small", e.ModelId); + + Assert.Null(e.CreatedAt); + DateTimeOffset createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); + e.CreatedAt = createdAt; + Assert.Equal(createdAt, e.CreatedAt); + + Assert.Null(e.AdditionalProperties); + AdditionalPropertiesDictionary props = new(); + e.AdditionalProperties = props; + Assert.Same(props, e.AdditionalProperties); + } + + [Fact] + public void Serialization_Roundtrips() + { + foreach (int length in Enumerable.Range(0, 64).Concat(new[] { 10_000 })) + { + bool[] bools = new bool[length]; + Random r = new(42); + for (int i = 0; i < length; i++) + { + bools[i] = r.Next(2) != 0; + } + + BitArray vector = new BitArray(bools); + BinaryEmbedding e = new(vector); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal($$"""{"$type":"binary","vector":"{{string.Concat(vector.Cast().Select(b => b ? '1' : '0'))}}"}""", json); + + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector, result.Vector); + } + } + + [Fact] + public void Derialization_SupportsEncodedBits() + { + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize( + """{"$type":"binary","vector":"\u0030\u0031\u0030\u0031\u0030\u0031"}""", + TestJsonSerializerContext.Default.Embedding)); + + Assert.Equal(new BitArray(new[] { false, true, false, true, false, true }), result.Vector); + } + + [Theory] + [InlineData("""{"$type":"binary","vector":"\u0030\u0032"}""")] + [InlineData("""{"$type":"binary","vector":"02"}""")] + [InlineData("""{"$type":"binary","vector":" "}""")] + [InlineData("""{"$type":"binary","vector":10101}""")] + public void Derialization_InvalidBinaryEmbedding_Throws(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs index 45fcce8ba63..c3809782006 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs @@ -14,7 +14,7 @@ public class EmbeddingTests public void Embedding_Ctor_Roundtrips() { float[] floats = [1f, 2f, 3f]; - UsageDetails usage = new(); + AdditionalPropertiesDictionary props = []; var createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); const string Model = "text-embedding-3-small"; @@ -35,6 +35,32 @@ public void Embedding_Ctor_Roundtrips() Assert.Same(floats, array.Array); } + [Fact] + public void Embedding_Byte_SerializationRoundtrips() + { + byte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"uint8","vector":"AQID"}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + + [Fact] + public void Embedding_SByte_SerializationRoundtrips() + { + sbyte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"int8","vector":[1,2,3]}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + #if NET [Fact] public void Embedding_Half_SerializationRoundtrips() @@ -43,7 +69,7 @@ public void Embedding_Half_SerializationRoundtrips() Embedding e = new(halfs); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"halves","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float16","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -57,7 +83,7 @@ public void Embedding_Single_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"floats","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float32","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -70,7 +96,7 @@ public void Embedding_Double_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"doubles","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float64","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs new file mode 100644 index 00000000000..4233e5cdbe1 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI.Utilities; + +public static class AIJsonSchemaTransformCacheTests +{ + [Fact] + public static void NullOptions_ThrowsArgumentNullException() + { + Assert.Throws(() => new AIJsonSchemaTransformCache(transformOptions: null!)); + } + + [Fact] + public static void EmptyOptions_ThrowsArgumentException() + { + Assert.Throws(() => new AIJsonSchemaTransformCache(transformOptions: new())); + } + + [Fact] + public static void TransformOptions_ReturnsExpectedValue() + { + AIJsonSchemaTransformOptions options = new() { ConvertBooleanSchemas = true }; + AIJsonSchemaTransformCache cache = new(options); + Assert.Same(options, cache.TransformOptions); + } + + [Fact] + public static void NullFunction_ThrowsArgumentNullException() + { + AIJsonSchemaTransformCache cache = new(new() { ConvertBooleanSchemas = true }); + Assert.Throws(() => cache.GetOrCreateTransformedSchema(function: null!)); + } + + [Fact] + public static void NullResponseFormat_ThrowsArgumentNullException() + { + AIJsonSchemaTransformCache cache = new(new() { ConvertBooleanSchemas = true }); + Assert.Throws(() => cache.GetOrCreateTransformedSchema(responseFormat: null!)); + } + + [Fact] + public static void FunctionSchema_ReturnsExpectedResults() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + + AIFunction func = AIFunctionFactory.Create((int x, int y) => x + y); + JsonElement transformedSchema = cache.GetOrCreateTransformedSchema(func); + Assert.True(transformedSchema.TryGetProperty("myAwesomeKeyword", out _)); + + JsonElement transformedSchema2 = cache.GetOrCreateTransformedSchema(func); + Assert.Equal(transformedSchema, transformedSchema2); + } + + [Fact] + public static void ChatResponseFormat_ReturnsExpectedResults() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + + JsonElement schema = JsonDocument.Parse("{}").RootElement; + ChatResponseFormatJson responseFormat = ChatResponseFormat.ForJsonSchema(schema); + JsonElement? transformedSchema = cache.GetOrCreateTransformedSchema(responseFormat); + Assert.NotNull(transformedSchema); + Assert.True(transformedSchema.Value.TryGetProperty("myAwesomeKeyword", out _)); + + JsonElement? transformedSchema2 = cache.GetOrCreateTransformedSchema(responseFormat); + Assert.Equal(transformedSchema, transformedSchema2); + } + + [Fact] + public static void ChatResponseFormat_NullFormatReturnsNullSchema() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + JsonElement? transformedSchema = cache.GetOrCreateTransformedSchema(ChatResponseFormat.Json); + Assert.Null(transformedSchema); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs index c4141a4bf0d..2d1967b11c1 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs @@ -566,6 +566,231 @@ public static void CreateFunctionJsonSchema_InvokesIncludeParameterCallbackForEv Assert.Contains("fifth", schemaString); } + [Fact] + public static void TransformJsonSchema_ConvertBooleanSchemas() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": { "not": true } } }, + "baz": { } + }, + "required": ["foo"] + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + ConvertBooleanSchemas = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_RequireAllProperties() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false }, "required": ["x"] }, + "baz": true + }, + "required": ["foo", "bar", "baz"] + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + RequireAllProperties = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_DisallowAdditionalProperties() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false }, "additionalProperties": false }, + "baz": true + }, + "required": ["foo"], + "additionalProperties": false + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + DisallowAdditionalProperties = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_UseNullableKeyword() + { + JsonElement schema = JsonDocument.Parse(""" + { + "type": ["object","null"], + "properties" : { + "foo": { "type": ["null","string"] }, + "bar": { "type":"object", "properties": { "x": false } }, + "baz": { "type" : ["string","array","null"] } + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "type": "object", + "properties" : { + "foo": { "type": "string", "nullable": true }, + "bar": { "type":"object", "properties": { "x": false } }, + "baz": { "type" : ["string","array","null"] } + }, + "required": ["foo"], + "nullable": true + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + UseNullableKeyword = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Theory] + [MemberData(nameof(TestTypes.GetTestDataUsingAllValues), MemberType = typeof(TestTypes))] + public static void TransformJsonSchema_ValidateWithTestData(ITestData testData) + { + // Stress tests the schema generation method using types from the JsonSchemaExporter test battery. + + JsonSerializerOptions options = testData.Options is { } opts + ? new(opts) { TypeInfoResolver = TestTypes.TestTypesContext.Default } + : TestTypes.TestTypesContext.Default.Options; + + JsonTypeInfo typeInfo = options.GetTypeInfo(testData.Type); + AIJsonSchemaCreateOptions? createOptions = typeInfo.Properties.Any(prop => prop.IsExtensionData) + ? new() { DisallowAdditionalProperties = false } // Do not append additionalProperties: false to the schema if the type has extension data. + : null; + + JsonElement schema = AIJsonUtilities.CreateJsonSchema(testData.Type, serializerOptions: options, inferenceOptions: createOptions); + + int totalSchemaNodes = 0; + AIJsonSchemaTransformOptions transformOptions = new() + { + ConvertBooleanSchemas = true, + RequireAllProperties = true, + DisallowAdditionalProperties = true, + UseNullableKeyword = true, + TransformSchemaNode = (context, schema) => + { + totalSchemaNodes++; + var schemaObj = Assert.IsType(schema); + schemaObj.Add("myAwesomeKeyword", (JsonNode)42); + return schemaObj; + } + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, transformOptions); + Assert.True(totalSchemaNodes > 0, "TransformSchema was not invoked."); + + int totalSchemaNodes2 = 0; + transformOptions = new() + { + TransformSchemaNode = (context, schema) => + { + totalSchemaNodes2++; + var schemaObj = Assert.IsType(schema); + Assert.Contains("myAwesomeKeyword", schemaObj); + if (schemaObj.TryGetPropertyValue("properties", out JsonNode? props)) + { + Assert.Contains("required", schemaObj); + Assert.Contains("additionalProperties", schemaObj); + Assert.Equal(((JsonArray)schemaObj["required"]!).Count, ((JsonObject)props!).Count); + } + + if (schemaObj.TryGetPropertyValue("type", out JsonNode? type) && type is JsonArray typeArray) + { + Assert.DoesNotContain("null", typeArray); + } + + return schemaObj; + } + }; + + AIJsonUtilities.TransformSchema(transformedSchema, transformOptions); + Assert.Equal(totalSchemaNodes, totalSchemaNodes2); + } + + [Fact] + public static void TransformJsonSchema_InvalidOptions_ThrowsArgumentException() + { + JsonElement schema = JsonDocument.Parse("{}").RootElement; + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions: null!)); + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions: new())); + } + + [Theory] + [InlineData("null")] + [InlineData("42")] + [InlineData("[1,2,3]")] + [InlineData("""{"properties":{"x": 42 }}""")] + [InlineData("""{"oneOf":[42]}""")] + public static void TransformJsonSchema_InvalidInput_ThrowsArgumentException(string invalidSchema) + { + JsonElement schema = JsonDocument.Parse(invalidSchema).RootElement; + AIJsonSchemaTransformOptions transformOptions = new() { ConvertBooleanSchemas = true }; + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions)); + } + private class DerivedAIContent : AIContent { public int DerivedValue { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs index 788b8568607..26cd380ec83 100644 --- a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net.Http; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Azure; using Azure.AI.Inference; @@ -32,6 +33,19 @@ public void AsIChatClient_InvalidArgs_Throws() Assert.Throws("defaultModelId", () => client.AsIChatClient(" ")); } + [Fact] + public async Task NullModel_Throws() + { + ChatCompletionsClient client = new(new("http://localhost/some/endpoint"), new AzureKeyCredential("key")); + IChatClient chatClient = client.AsIChatClient(modelId: null); + + await Assert.ThrowsAsync(() => chatClient.GetResponseAsync("hello")); + await Assert.ThrowsAsync(() => chatClient.GetStreamingResponseAsync("hello").GetAsyncEnumerator().MoveNextAsync().AsTask()); + + await Assert.ThrowsAsync(() => chatClient.GetResponseAsync("hello", new ChatOptions { ModelId = null })); + await Assert.ThrowsAsync(() => chatClient.GetStreamingResponseAsync("hello", new ChatOptions { ModelId = null }).GetAsyncEnumerator().MoveNextAsync().AsTask()); + } + [Fact] public void AsIChatClient_ProducesExpectedMetadata() { @@ -76,54 +90,54 @@ public void GetService_SuccessfullyReturnsUnderlyingClient() Assert.Null(pipeline.GetService("key")); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task BasicRequestResponse_NonStreaming(bool multiContent) - { - const string Input = """ - { - "messages": [{"role":"user", "content":"hello"}], - "max_tokens":10, - "temperature":0.5, - "model":"gpt-4o-mini" - } - """; + private const string BasicInputNonStreaming = """ + { + "messages": [{"role":"user", "content":"hello"}], + "max_tokens":10, + "temperature":0.5, + "model":"gpt-4o-mini" + } + """; - const string Output = """ + private const string BasicOutputNonStreaming = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "created": 1727888631, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ { - "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", - "object": "chat.completion", - "created": 1727888631, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello! How can I assist you today?", - "refusal": null - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 8, - "completion_tokens": 9, - "total_tokens": 17, - "prompt_tokens_details": { - "cached_tokens": 0 + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "refusal": null }, - "completion_tokens_details": { - "reasoning_tokens": 0 - } - }, - "system_fingerprint": "fp_f85bea6784" + "logprobs": null, + "finish_reason": "stop" } - """; + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 9, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "system_fingerprint": "fp_f85bea6784" + } + """; - using VerbatimHttpHandler handler = new(Input, Output); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BasicRequestResponse_NonStreaming(bool multiContent) + { + using VerbatimHttpHandler handler = new(BasicInputNonStreaming, BasicOutputNonStreaming); using HttpClient httpClient = new(handler); using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); @@ -153,50 +167,50 @@ [new ChatMessage(ChatRole.User, "hello".Select(c => (AIContent)new TextContent(c Assert.Equal(17, response.Usage.TotalTokenCount); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task BasicRequestResponse_Streaming(bool multiContent) - { - const string Input = """ - { - "messages": [{"role":"user", "content":"hello"}], - "max_tokens":20, - "temperature":0.5, - "stream":true, - "model":"gpt-4o-mini"} - """; + private const string BasicInputStreaming = """ + { + "messages": [{"role":"user", "content":"hello"}], + "max_tokens":20, + "temperature":0.5, + "stream":true, + "model":"gpt-4o-mini"} + """; - const string Output = """ - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + private const string BasicOutputStreaming = """ + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[],"usage":{"prompt_tokens":8,"completion_tokens":9,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[],"usage":{"prompt_tokens":8,"completion_tokens":9,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} - data: [DONE] + data: [DONE] - """; + """; - using VerbatimHttpHandler handler = new(Input, Output); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BasicRequestResponse_Streaming(bool multiContent) + { + using VerbatimHttpHandler handler = new(BasicInputStreaming, BasicOutputStreaming); using HttpClient httpClient = new(handler); using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); @@ -230,6 +244,420 @@ [new ChatMessage(ChatRole.User, "hello".Select(c => (AIContent)new TextContent(c } } + [Fact] + public async Task IChatClient_WithNullModel_ChatOptions_WithNotNullModel_NonStreaming() + { + using VerbatimHttpHandler handler = new(BasicInputNonStreaming, BasicOutputNonStreaming); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + + var response = await client.GetResponseAsync("hello", new ChatOptions + { + ModelId = "gpt-4o-mini", + MaxOutputTokens = 10, + Temperature = 0.5f, + }); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task IChatClient_WithNullModel_ChatOptions_WithNotNullModel_Streaming() + { + using VerbatimHttpHandler handler = new(BasicInputStreaming, BasicOutputStreaming); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", new ChatOptions + { + ModelId = "gpt-4o-mini", + MaxOutputTokens = 20, + Temperature = 0.5f, + })) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type": "object","required": ["personName"],"properties": {"personName": {"description": "The person whose age is being requested","type": "string"}}}}} + ], + "tool_choice":"auto", + "additional_property_from_raw_representation":42, + "additional_property_from_MEAI_options":42 + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + return azureAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json, + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["additional_property_from_MEAI_options"] = 42 + } + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type": "object","required": ["personName"],"properties": {"personName": {"description": "The person whose age is being requested","type": "string"}}}}} + ], + "tool_choice":"auto", + "additional_property_from_raw_representation":42, + "additional_property_from_MEAI_options":42, + "stream":true + } + """; + + const string Output = """ + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + return azureAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json, + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["additional_property_from_MEAI_options"] = 42 + } + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none" + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + Assert.Empty(azureAIOptions.Messages); + Assert.Null(azureAIOptions.Model); + Assert.Null(azureAIOptions.FrequencyPenalty); + Assert.Null(azureAIOptions.MaxTokens); + Assert.Null(azureAIOptions.NucleusSamplingFactor); + Assert.Null(azureAIOptions.PresencePenalty); + Assert.Null(azureAIOptions.Temperature); + Assert.Null(azureAIOptions.Seed); + Assert.Empty(azureAIOptions.StopSequences); + Assert.Empty(azureAIOptions.Tools); + Assert.Null(azureAIOptions.ToolChoice); + Assert.Null(azureAIOptions.ResponseFormat); + return azureAIOptions; + }, + ModelId = "gpt-4o-mini", + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none", + "stream":true + } + """; + + const string Output = """ + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + Assert.Empty(azureAIOptions.Messages); + Assert.Null(azureAIOptions.Model); + Assert.Null(azureAIOptions.FrequencyPenalty); + Assert.Null(azureAIOptions.MaxTokens); + Assert.Null(azureAIOptions.NucleusSamplingFactor); + Assert.Null(azureAIOptions.PresencePenalty); + Assert.Null(azureAIOptions.Temperature); + Assert.Null(azureAIOptions.Seed); + Assert.Empty(azureAIOptions.StopSequences); + Assert.Empty(azureAIOptions.Tools); + Assert.Null(azureAIOptions.ToolChoice); + Assert.Null(azureAIOptions.ResponseFormat); + return azureAIOptions; + }, + ModelId = "gpt-4o-mini", + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + /// Converts an Extensions function to an AzureAI chat tool. + private static ChatCompletionsToolDefinition ToAzureAIChatTool(AIFunction aiFunction) + { + // Map to an intermediate model so that redundant properties are skipped. + var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema)!; + var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool)); + return new(new FunctionDefinition(aiFunction.Name) + { + Description = aiFunction.Description, + Parameters = functionParameters, + }); + } + + /// Used to create the JSON payload for an AzureAI chat tool description. + private sealed class AzureAIChatToolJson + { + [JsonPropertyName("type")] + public string Type { get; set; } = "object"; + + [JsonPropertyName("required")] + public List Required { get; set; } = []; + + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } = []; + } + [Fact] public async Task AdditionalOptions_NonStreaming() { @@ -279,10 +707,72 @@ public async Task AdditionalOptions_NonStreaming() PresencePenalty = 0.5f, Seed = 42, StopSequences = ["yes", "no"], - AdditionalProperties = new() + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + azureAIOptions.AdditionalProperties.Add("something_else", new BinaryData(JsonSerializer.SerializeToUtf8Bytes("value1", typeof(object)))); + azureAIOptions.AdditionalProperties.Add("and_something_further", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(123, typeof(object)))); + return azureAIOptions; + }, + })); + } + + [Fact] + public async Task TopK_DoNotOverwrite_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user", "content":"hello"}], + "max_tokens":10, + "temperature":0.5, + "top_p":0.5, + "stop":["yes","no"], + "presence_penalty":0.5, + "frequency_penalty":0.75, + "seed":42, + "model":"gpt-4o-mini", + "top_k":40, + "something_else":"value1", + "and_something_further":123 + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); + + Assert.NotNull(await client.GetResponseAsync("hello", new() + { + MaxOutputTokens = 10, + Temperature = 0.5f, + TopP = 0.5f, + TopK = 20, // will be ignored because the raw representation already specifies it. + FrequencyPenalty = 0.75f, + PresencePenalty = 0.5f, + Seed = 42, + StopSequences = ["yes", "no"], + RawRepresentationFactory = (c) => { - ["something_else"] = "value1", - ["and_something_further"] = 123, + ChatCompletionsOptions azureAIOptions = new(); + azureAIOptions.AdditionalProperties.Add("top_k", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(40, typeof(object)))); + azureAIOptions.AdditionalProperties.Add("something_else", new BinaryData(JsonSerializer.SerializeToUtf8Bytes("value1", typeof(object)))); + azureAIOptions.AdditionalProperties.Add("and_something_further", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(123, typeof(object)))); + return azureAIOptions; }, })); } diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageEmbeddingGeneratorTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageEmbeddingGeneratorTests.cs new file mode 100644 index 00000000000..7ca945eb07b --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceImageEmbeddingGeneratorTests.cs @@ -0,0 +1,128 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Azure; +using Azure.AI.Inference; +using Azure.Core.Pipeline; +using Microsoft.Extensions.Caching.Distributed; +using Microsoft.Extensions.Caching.Memory; +using Xunit; + +#pragma warning disable S103 // Lines should not be too long + +namespace Microsoft.Extensions.AI; + +public class AzureAIInferenceImageEmbeddingGeneratorTests +{ + [Fact] + public void AsIEmbeddingGenerator_InvalidArgs_Throws() + { + Assert.Throws("imageEmbeddingsClient", () => ((ImageEmbeddingsClient)null!).AsIEmbeddingGenerator()); + + ImageEmbeddingsClient client = new(new("http://somewhere"), new AzureKeyCredential("key")); + Assert.Throws("defaultModelId", () => client.AsIEmbeddingGenerator(" ")); + + client.AsIEmbeddingGenerator(null); + } + + [Fact] + public void AsIEmbeddingGenerator_OpenAIClient_ProducesExpectedMetadata() + { + Uri endpoint = new("http://localhost/some/endpoint"); + string model = "amazingModel"; + + ImageEmbeddingsClient client = new(endpoint, new AzureKeyCredential("key")); + + IEmbeddingGenerator> embeddingGenerator = client.AsIEmbeddingGenerator(model); + var metadata = embeddingGenerator.GetService(); + Assert.Equal("az.ai.inference", metadata?.ProviderName); + Assert.Equal(endpoint, metadata?.ProviderUri); + Assert.Equal(model, metadata?.DefaultModelId); + } + + [Fact] + public void GetService_SuccessfullyReturnsUnderlyingClient() + { + var client = new ImageEmbeddingsClient(new("http://somewhere"), new AzureKeyCredential("key")); + var embeddingGenerator = client.AsIEmbeddingGenerator("model"); + + Assert.Same(embeddingGenerator, embeddingGenerator.GetService>>()); + Assert.Same(client, embeddingGenerator.GetService()); + + using IEmbeddingGenerator> pipeline = embeddingGenerator + .AsBuilder() + .UseOpenTelemetry() + .UseDistributedCache(new MemoryDistributedCache(Options.Options.Create(new MemoryDistributedCacheOptions()))) + .Build(); + + Assert.NotNull(pipeline.GetService>>()); + Assert.NotNull(pipeline.GetService>>()); + Assert.NotNull(pipeline.GetService>>()); + + Assert.Same(client, pipeline.GetService()); + Assert.IsType>>(pipeline.GetService>>()); + } + + [Fact] + public async Task GenerateAsync_ExpectedRequestResponse() + { + DataContent dotnetPng = new(ImageDataUri.GetImageDataUri()); + + const string Input = """ + { + "input":[{"image":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAcgAAAHICAMAAAD9f4rYAAAAS1BMVEVRK9RRK9T///\u002BolenTyvTp5fpnRdl8YN\u002B\u002Br\u002B708v1cONedh\u002Be\u002Bru5nRtl9YN6HbeKyouzJvfKSeuSzou2\u002Br\u002B9yU9ze1/dcONbe2PcNfWisAAAAAXRSTlP\u002BGuMHfQAAB79JREFUeNrs0QENAAAMw6Ddv\u002Bn7aMACOwomskFkhMgIkREiI0RGiIwQGSEyQmSEyAiRESIjREaIjBAZITJCZITICJERIiNERoiMEBkhMkJkhMgIkREiI0RGiIwQGSEyQmSEyAiRESIjREaIjBAZITJCZITICJERIiNERoiMEBkhMkJkhMgIkREiI0RGiIwQGSEyQmSEyAiRESIjREaIjBAZITJCZITICJERIiNERoiMEBkhMkJkhMgIkREiI0RGiIwQGSEyQmSEyAiRESIjREaIjBAZITJCZITICJERIiNERoiMEBkhMkJkhMgIkREiI0RGiIwQGSEyQmSEyAiRESIjREaIjBAZITJCZITICJERIiNERoiMEBkhMkJkhMgIkREiI0RGiIwQGSEyQmSEyAiRESKfnTvMTRyGoiisF5K2SYZhKKX7X\u002BpEeuov7Ngxorp\u002BOmcH9KssLnISJCCDBGSQgAwSkEECMkhABgnIIAEZJCCDBGSQgAwSkEECMkhABgnIIAEZJCCDBGSQgAwSkEECMkhABgnIIAEZJCCDBGSQgAwSkEECMkhABgnIIAEZJCCDBGSQgAwSkEECMki/DzkNqUZr7H146M0ynYZnmgof4cn\u002B2BPpQA6rFQMymxDk/GalgMwmBDlcrRSQ2ZQgh79WCMhsUpDTYvsBmU0Kcvhn\u002BwGZTQuydLgCmU0MsjAmgcwmBlkYk0BmU4PcH5NAZlOD3D9cgcwmBzlcLB\u002BQ2fQg98YkkNn0IPfGJJDZBCF3xiSQ2RQhvy3XKyDnsboP\u002B\u002Bk6FpoT/wZjodWeSBEyPyZfATnaKxqHh072yiQhj4xJID1JyCN/XCA9TcgDYxJITxRyXqwyID1RyPoxCaSnClk9JoH0NCDH9jEJpKcBeR\u002BaPzeQngbk5do8JoH0NCA/35vHJJCeBuRqY0Ly0yoC0tOAPNm5dUwC6alA2q1xTALpaUBuYsvUNiaB9DQgP8w9Gq59AOnpQNq1aUwC6QlBnueWMQmkJwRpa8uYBNJTgrSx4doHkJ4UZMuYBNKTgkzeVvyy3YD0tCAbxiSQnhZkw5gE0hODtNvRMQmkpwa5zEOtiwekpwZpl4NjEkhPDvLomATS04M8z4fGJJCeHqSth95uBqQnCGnjkTEJpKcIeT8yJoH0FCEPjUkgPUnI5C91d0v2a08sf1p9QJp34JprM2S5dgcgf/qqHpNAeqKQS/W1DyA9Ucj6MQmkpwpZPSaB9GQhz3PdmATSk4W0U90zBEB6upD2XXW4AukJQ9aNSSA9YUi71YxJID1lyGWqGJNAesqQVYcrkJ40pF3LbzcD0tOGXMpjEkhPG9LW4pgE0hOHLP9S9zTkPNW1Wn1APnSeC28344aApw5pp8KYBNKTh7TCmATS04csjEkgPX1Iu\u002B2OSSC9DiCXae8ZAiC9DiDtsjcmgfR6gNwdk0B6XUDujUkgvS4gbc3/ZAak1wekjdkxCaTXCeQ9OyaB9DqBtFPuVdlAer1AZsckkF4vkPaeGZNAet1A2i09JoH0\u002BoHMXvu4A7nVD6RdMmPyDcitjiDTYxJIryfI85xkWIDc6gnS1vS1DyC3uoK0MTkmZyDN\u002BoJMj8kJSLO\u002BINNjcgTSrDPIZUpIfAFp1hlk8nDlaN3qDTL1KiW\u002BtW51B7nMQKbqDtJWIP\u002BzdwerDcNQEEUZWbIqG9XESev8/5d2EQol7wXcZBSwmLv3Zg54oYXkdTxIREE6HRCyFkHa2JDbfEohlHj5xINehsQgSBsXchtK\u002BC2tcHsdEt\u002BCNFEhx7Tj0XICZBakiQk53gvFCTYCJM5EyOv4nzbs6diQowW6wMaAnBIBsuGVEMeG3Hl9NQMSWZAmFmQO\u002Bx7WpUDiJMhbfEh/2hkmCmQtgkQbyOB2gokCiVmQQAvIHNwSTBxIREE2gVyCH0wkyCrIJpBrMLWFxCDIVr/W90JOSZANIMfgdoWJBYksSD6kx\u002BOft/IgcRZkA0h/owoTD3IqgqRD\u002BqteYCJCYhEkHdJdNVWYmJCIguRD2pXKF2xUyFoESYc0MyXXkQqJWZANILH\u002BNYoVfvNw34KnmwenCQ/Kw4vlvUt4n7aKDwms8aZYPjLU2\u002BJDAlte1jxCvbUbpOohQXaSIDtJkJ0kyE4SZCcJspME2UmC/GGPDmQAAAAABvlb36M9hRBHIo5EHIk4EnEk4kjEkYgjEUcijkQciTgScSTiSMSRiCMRRyKORByJOBJxJOJIxJGIIxFHIo5EHIk4EnEk4kjEkYgjEUciYo8OZAAAAAAG\u002BVvf4yuFRE6InBA5IXJC5ITICZETIidEToicEDkhckLkhMgJkRMiJ0ROiJwQOSFyQuSEyAmREyInRE6InBA5IXJC5ITICZETIidEToicEDkhckLkhMgJkRMiJ0ROiJwQOSFyQuSEyAmREyInRE6InBA5IXJC5ITICZETIidEToicEDkhckLkhMgJkRMiJ0ROiJwQWXt0QAMAAIAwyP6p7cFOBRBFIopEFIkoElEkokhEkYgiEUUiikQUiSgSUSSiSESRiCIRRSKKRBSJKBJRJKJIRJGIIhFFIopEFIkoElEkokhEkYgiEUUiikQUiSgSUSSiSESRiCIRRSKKRBSJKBJRJKJIRJGIIhFFIopEFIkoElEkokjEgjh2WnxgwCuWdQAAAABJRU5ErkJggg=="}], + "encoding_format":"base64", + "model":"embed-v-4-0" + } + """; + + const string Output = """ + { + "id":"9da7c0f0-4b9d-46d9-9323-f10f46977493", + "object":"list", + "data": + [ + { + "index":0, + "object":"embedding", + "embedding":"AADkPAAA6bsAAD68AAAnOwAAYbwAAFa8AABfvQAAqzsAAGy8AABwvAAAyDwAALo9AAC6vAAAID0AAGE8AAA2PAAA4TsAABU9AAC0PAAAqzwAADw8AAAaPQAA2LwAANa8AACoOgAA4DsAAA48AAC9PAAAdz0AAIC8AACGvAAA+7oAAEo7AABMuwAAab0AAFc9AAA2OwAAob0AAPO8AABGOwAAoLwAAKQ6AACHvAAAX70AAJQ8AAA/OwAAtbwAAME7AADMvAAA2DwAABQ9AABZPQAAd7wAAD+9AAAquwAAgTwAABE9AABFPQAALbwAAEk9AAC8ugAABj0AAI27AAAWPQAAMrwAAE88AABnvAAAZbwAAMK8AAAhPQAAPr0AAAg8AABcOgAA/jwAANE8AACvvAAAEbwAALy8AAAIvQAAe7wAAGW8AAAPPQAAFTsAALc5AACrOwAAirwAALa8AACLvAAACLsAABa7AAC+uAAAljwAAMS9AAC0vAAAhz0AAJw7AAAgvQAAxjwAAMK8AACMPAAAdz0AALC8AADIPAAArjsAAGG8AAATvAAAkrsAALs8AAAWPAAAxDwAADK9AAAAvQAAmTwAAIK9AADZPAAAmjwAABG9AAARuwAA/zwAAGO8AAC3PAAAGTwAACC8AAAtPAAAArwAAGG7AAC4PAAA/7sAAKG8AACdOwAA8DwAAJo7AAC8PAAAST0AAAI8AABnvAAAXTwAABc9AACSPAAAMjsAAPc7AABSvAAATLsAAKa8AAB1PAAAA70AAC87AAASvAAA/DwAADC7AABfvAAAYbwAAGW9AADlOwAANzwAAFc8AADEPAAAyrsAAMM8AAATPQAA3DwAABu8AAB+uQAAKj0AADS7AACkOwAAhD0AACK8AABIvQAAaboAALu8AADtOwAAoDwAAI88AACQPAAAmjwAAEy8AAC2OwAAtTwAAE68AACGvQAA0LsAAJM7AAAUvQAA17wAAEg9AABhOwAAVjwAALg8AACHvQAA5DwAACI9AACLPAAA4zsAAOk7AADOPAAA/7wAAPe8AAAGPQAAYTwAAEo9AAA/PAAA47wAAEq8AADgvAAAybsAAPk8AAA7vQAA3zsAAP87AAAUvAAAKjwAAKA8AAATvQAAcrwAAGm7AADlvAAAprwAAJM7AACivAAABr0AAEu7AAAxuwAAjD0AAMu8AAA7vQAATjwAADo8AADfOgAAFboAACA9AAA2OwAAZDwAAOo7AABBvAAAKzsAAJK8AAC7vAAAFL0AAO47AAADvQAARj0AAJS8AADLuwAA5bwAAKa7AAAGPQAA8bwAAKG9AAC/vAAADrwAAKO8AACDOgAAr7wAABM9AAD0uwAAQr0AABs9AAC2PAAAOLwAAM88AADSPAAAqzsAAOm7AABMPAAAGz0AAHU8AAAAvQAAULwAAAa9AADavAAAgzoAAKk8AABVvAAAPboAAHU8AACKvAAAgbwAADI8AAALuwAAIb0AAKi8AABxvAAAUz0AAJk8AAAnvQAA3zwAAMM7AAAVPAAA0bwAAME6AADCuwAAIrwAAMs8AACbPAAArbwAAGG8AAChOwAAEL0AAIQ7AADePAAADr0AADE9AAAbvQAAprwAAK+8AAARvAAAWrwAAL+8AAALPAAAWTwAAJ86AACGOwAAU70AACm8AAAJPQAA+LwAAKC8AABtvAAAtLwAALQ7AACmvAAAAj0AALW8AAA0PQAAhjwAAEa6AAAfPQAAirwAAOa7AABDPAAAqLwAANM8AAAGvQAA4DsAANO7AAAdvAAA7TwAACM7AACfvAAASDsAABs8AACxPAAAVzwAAEy9AAAxPAAAmb0AALw8AAAZvAAAiLwAALY8AAB3vAAA9zwAAJs8AAAkvAAAOz0AAMo7AABLPQAAwbsAAN47AABGuQAAl7oAAG08AACJvAAAZ7oAALw8AACavAAA37wAAKA7AAAgvAAANb0AAGA8AAAhPQAANz0AAMq7AADGvAAAlTwAABI9AABhuwAAkbsAAIY7AADauwAAtDwAABk8AAD7PAAAiDwAAPG8AACwvQAAn70AAFI8AACqugAAn7wAAGA9AAA0vQAAmrwAACo9AACCOwAAoTsAAIE9AABwOwAAAr0AANc8AAAbvAAAjDwAABe9AAAPvQAA07wAACG8AADBOwAAeDwAAAg9AAB0PAAAm7wAAEW6AACaugAADr0AANY8AAD1vAAA5zsAAKK9AAAXOwAAPr0AAAA9AAD3uwAAG7wAABW9AAAOPQAAMrwAAIA7AADdPAAAEb0AAGM8AAAjvQAAUDoAABI9AAD/PAAAHL0AAKM8AACbOQAAlbwAAAO9AACqPAAAAr0AAIy7AACCvAAAZjwAAGO8AAC9PQAA7DsAAJ88AAByOwAAmrsAAD+8AAArvAAA37wAAPo8AAAkvAAAL7sAACO9AAAnvQAA9DwAAJY8AACxPAAAeTwAAFO8AAAFvQAAHzwAADe7AACmPAAAKD0AAHM9AAAgvAAAmrwAALy8AAC/OwAA3LwAAG06AAAfOwAA/7sAALE9AADBPAAAtrsAAKI8AACZuwAAgrwAAES9AADcuwAAsjsAAJE8AABWPAAAK70AAEU8AABEPAAAMbwAAK+7AACcvAAARLsAABK6AAAiPAAAEbwAANG7AAChPAAAzzwAAMs6AAAFPQAA2TsAACG9AAB1PAAAsrwAAC29AABMPAAAzzwAANI8AADfvAAAm7wAAC29AACLuwAAHTwAALq8AAAcuwAA07wAAHm8AACxvAAA7LwAAK06AAA4PQAA7LsAAKC7AAAvuwAAKrwAAC68AABtPAAAtjwAAC+8AAAJvQAATLwAALE7AACCvAAApjwAAKE8AAC4vAAAjDwAACS9AAD3PAAAHz4AACe9AAB7vQAAET0AAII6AAC2OwAAyzwAANY7AAB+PQAAuDwAAME8AADMugAAAjwAANA7AAAgvAAAFT4AAPe7AAAPvQAALrwAAJQ5AAArOwAAFjwAAKe8AAD4uwAAGTwAACQ8AAAJPAAAZTwAAJa8AACgOwAANjsAAJk8AAC7OwAAdzwAAPG7AACfvAAAtjwAAFq8AAAMPQAAMDwAAHu9AAC6vAAAVT0AAKo7AACOPAAAoTsAANc8AAAXPQAAbDwAAKi7AABVOgAA5zwAAHU8AADCvAAAyjwAAAa9AADqOwAAmbwAALq7AAA+vAAAjDwAAB+9AAAqvQAAir0AAFo9AAA+PQAAgrsAANM8AAAhPAAAhbwAAAU8AACavAAAuLwAAKa8AACqPAAAI7wAAHG8AABFPAAAgToAAIy8AAAkuwAAjrwAAA49AACpPAAACz0AABC9AAAbvAAAWjwAAPI7AAAoPAAAJjoAAK26AAAXOwAADzwAAC+9AAC4vAAAIL0AAIk8AABhPAAAPj0AAHI7AAAUvQAALT0AACG8AAByPAAADD0AANk8AAC/vAAA4bwAAGu8AAC1vAAA0jsAALc8AAChvAAAT7wAAMu8AACOvAAA4bwAAHg8AAD1PAAACz0AAB08AAAXPAAAPr0AAIG6AAAFuwAAKTwAAI27AABPPAAAmzsAAOC8AAAbPQAAp7oAAGq8AABdOgAAzDwAANe8AAAdvQAALjoAABU9AAATPQAA0rsAAAc9AAD7PAAATLwAALA6AAAruwAAX7sAABK9AAC7PAAAErwAACG8AAC3OgAAkzwAAMw7AAAEOwAAqjwAAEW7AAAHPQAA6rsAAES8AACCPQAARj0AAGY8AABGPAAAdLwAANE8AAD1vAAAGzsAAEQ6AACuuwAAFb0AAIE8AAA4PAAAlbsAAH68AAACOwAAsjwAAKE8AAAoPAAAhDsAAME8AAD7uwAAkr0AAFq9AAC4OwAAsjwAADA8AACCvAAAbbwAAAs9AACWvAAAEzwAALS8AAAgPQAAd7wAAO42AABWvQAAHLwAAPG6AAAAPAAAFz0AAME7AAAoOwAAULsAANo8AABRuwAAiDwAABw8AADVuwAA+rsAAAo9AAAavAAAMDwAANe8AAD+vAAAibwAAJC8AABfOwAAtTwAAIE8AADmOwAAgLwAAMS8AABwPAAAAb0AALS8AAAqvQAANDwAAOU8AACWvAAAzjwAABG7AACouwAAJr0AAIM7AAAZvQAA0boAAFi8AABPPAAAnzgAAIE8AACbvAAAFb0AABY8AAC+OwAA5DwAAJa6AACkPAAAITwAAGE6AABtPAAAMb0AADg9AAAEPQAAnDwAAJ08AAABPAAAursAAHc8AAAFvAAA5ToAAD28AAAAPAAAazwAADQ7AADqPAAAA70AAFO7AAA6vAAAAj4AAEg6AADhPAAAELwAAFm9AACIvAAAxTwAACQ8AADkOwAAbrwAALq7AACGPAAAIL0AAGE8AADMPAAAOr0AACM9AACMPAAAKrsAAAY8AAAhPAAAKz0AACe9AACOvAAAa7wAACG9AABKuwAASrwAAI+8AAApvQAA+LsAAPe8AADGuwAAgroAADe7AACvuwAATz0AAMQ6AACFvAAAMLwAACg9AAADvQAAtTwAALa8AACuPAAAI70AAJI8AAAauwAAZbwAAA89AADWvAAAqDwAAAm9AAAAPQAAEDwAAOA8AAAxvQAAYzwAAB87AADhOgAAwrsAAOA8AAA3vQAA1jwAAKi8AAB1uwAAGb0AAJo8AABmPAAAPLwAAMI7AAC4PAAAmj0AAFc8AADcOgAAe7wAAH47AABdOwAAlrwAAPO8AAB5PQAAijsAABU8AAAOvQAAkTwAABK8AAC4PAAAZLwAAK68AACRvAAAwzwAAKq8AABWvQAA4DsAAKC8AACUPAAAm7wAAJO8AAAMuQAAwrsAAAk8AABdvQAAkrwAACQ8AAAoNwAApDwAABQ8AAAVPAAAH7wAAFK8AAAGPAAAkrsAAIA8AADGPAAAbrwAALc8AABxPAAApDwAABy8AAAZPQAAk7wAAMW8AABhvAAAPLwAAEI8AAB5PAAAxrwAAFi7AADwvAAAUL0AAAk9AABZOwAAED0AALY8AAB5PAAAmzwAAFM9AAAwPQAAsToAAPA6AADOvAAAMLsAAHO8AADQuAAAqLwAANc7AAA4PAAA3DsAAK48AAAdPAAAH7wAACQ7AAD5OwAAo7sAACY8AACrPAAATzwAAL68AAC9PAAA8DwAABI7AADeOwAAFL0AAAC9AACEOwAAITsAAJI8AADtuwAA8LsAANa8AACvvAAAI70AAAG9AABmOwAAd7wAAIE8AAA6vQAAvzwAAEK9AAD0vAAA/zwAAPU8AACVPAAAET0AAAU7AAAfOwAANroAAKm8AAAUvQAAyLsAAAa9AAAUvAAAErwAAII7AAAFPQAAALsAAC08AAA0uwAAgTwAAIu7AADRvAAADzwAAKA7AABDvAAAirsAALo8AAB3vAAAOLwAACO9AADEPAAA7jwAADg9AAAiPQAAqzcAANA8AAAuPAAAODwAAAW8AACNvAAAIjwAANC8AAAmvQAAoTwAAAc9AACHvAAABjsAAI68AADZPAAAobsAAIi9AADsvAAABrsAAAm8AABkOwAACDwAAIY8AABQvAAAmTwAABE9AAAFvQAABzwAAF08AACoPAAAzjwAAL49AAAfPAAAkbwAALQ6AAByvAAAcD0AAN+6AACTvAAAkDsAAK66AAC0PAAAkzoAAHy8AAAiOwAADDwAAIG9AAAmvAAACrsAADU9AAAjuAAAjbwAAPc8AACNOwAABbwAAMG7AACIvAAAO7wAAL88AAD7vAAAXLwAADw4AAC2PAAAnbsAADs7AAAwvAAA0LwAAPG8AAAmPQAAz7oAAOa8AABhuwAA+jwAAFU8AADLuwAAtzwAAHA8AAA3vAAAdbwAAIG8AAC6PAAAiDkAANi7AADpuQAALrsAAL09AABauwAAMbwAAOG8AAA2OgAAejsAAGY8AAB/uwAACTsAADa7AAAGvAAASrwAAKG8AAC2OgAA3LoAABy8AACiPAAACD0AAPy8AACyvAAAIDsAAIi7AACwvAAA6rwAAMy8AAA0vQAALr0AAKS7AABgPAAASbwAAA69AAAnvQAApLwAAIE8AACUOgAAYbwAABo7AACfPAAADr0AACg9AAAAvAAAFzwAAIM7AAABOwAAujwAABS9AABqvAAAHLwAAHg8AAB3PQAAQ7wAAB08AAAIPQAAhLwAAHq8AAAfPQAAljwAAME7AAChOwAA5jgAAAy7AAALPAAAv7wAAA08AAC+uwAAzDwAAAQ9AACoPAAANTwAANi8AAAPPQAABj0AAM68AAB7uwAAIz0AAB29AAATuwAAjbsAAJ88AACfOwAAAj0AAHi8AAA9vAAAYbwAAMo8AADpPAAAAbwAABU7AAAgPAAA+jsAAAm8AABgPAAAIb0AAIK9AABwPAAAtzwAAFi7AAAmPAAAozwAAFW9AAAwvAAAFT0AAJm8AADjvAAAEjsAAFI8AAACvQAANrwAAEm7AACLuwAAITwAABu8AAD4uwAAyLwAAFw8AAA2PAAAVTwAANW8AADDPAAAMLwAACC7AADMPAAARTsAAA28AABkPAAArjwAADI8AAAEvQAAujsAAFY8AABavAAA9zwAAKI8AABVPAAA+7sAAOC8AACFPQAAjTsAAKg8AACpuwAAsjsAABU7AABRPAAAHL0AAEY8AAAhPAAAerwAAKS7AAAXOwAAkLsAAAA9AAAxPAAA4TwAACi8AADYOwAAu7sAAF68AABLPAAATL0AAEK9AADwuwAAjDsAADW6AACEPAAAv7wAAJa8AABQPQAAfLwAAAe8AAC9PAAAnTsAABM8AADQvAAAcjwAAP86AAA2vQAAKD0AAMQ8AADevAAAobwAAGE8AAB7PAAAjzwAAIY8AACkPAAA2joAAKY8AAAGPQAAc7sAALw8AAABPAAAebwAAAs9AAAoOwAAmjsAAH48AABZPAAAAjwAAIm9AAAGvQAAFTwAACo7AACLvAAArrwAAJS6AADnugAABj0AAAu8AADcvAAAvbwAAKE5AADePAAAqbwAAOw8AAA2vQAA7ToAAIG7AAA2vQAAC70AACk8AACIPAAAFr0AAKe6AAAZvQAArzwAAG48AABsPAAAAbsAAD89AACnPAAAAb0AAOu8AAAQPQAA5TwAALg8AAAbOwAAWbwAAJu7AAAJPAAA4TwAABm9AAD0OgAA07wAAPe7AAB/uwAAED0AADs8AADEOgAAhrsAAJM8AABLvQAAq7wAAL06AACfPAAAlDwAAIY9AABavAAAjDoAAAG9AABlPAAAjLsAALK8AADaPAAA4LsAAPA8AABMvAAAXTsAAOG6AAD6OgAAvjkAANC8AABxuwAAybsAAK+7AABsOwAA5zwAABW8AAC9PAAAiDwAAPg8AAAJOwAAATsAADs8AAAdPQAAeTsAACK8AADrvAAASTsAAKM8AADMPAAAU70AABK8AAD0uwAA0rwAAF08AAChNwAAbbwAAB28AACZPAAAlLwAAME8AABmvAAAhjsAAPA9AADSvAAABTwAAP48AAAVvAAAdzwAADY8AACGPQAA2DwAAC07AAC0ugAAwTsAAJC8AACdPAAAajwAAKE7AAAiPQAAzjsAAA69AACdvAAAIr0AAEi9AADBOgAAgLsAANU8AACpPAAAP7wAAPq8AAAfPAAACTwAAC49AABhPQAAsjwAAMy7AAB0PQAABb0AAAy9AAAhvQAAWL0AAHy8AAAjPQAAjDwAAGC8AACbvAAADT0AAK08AACivAAAF7wAAL+8AACTPAAAz7wAAPw7AABfvAAAt7wAALi8AAAvPQAAtrsAAJY7AAAKPQAAr7wAACS9AAC8PAAAm7wAALa8AADBvAAA3zsAAIk8AABmOwAAw7wAAPm7AAArPAAAvzsAAF+8AABPuwAAXzwAAK+8AAA3PQAAG7wAAIg8AAAXvAAAprwAADA8AADEvAAAorwAANa8AABePQAAJr0AACG8AAAcvAAAQ70AAPC8AACxPAAAOLsAAOc6AABYPAAAsLwAAN68AACXuwAALbwAAJu7AAD7uwAA2jsAALY7AACWPAAAoLwAALa8AACwuwAA/DsAAEy8AAAiPAAA5bwAAGk8AABnPAAADzwAAF27AAAGOwAAtrsAAIS7AAAqPQAAeLwAAAa9" + } + ], + "model":"embed-v4.0", + "usage": + { + "prompt_tokens":1012, + "completion_tokens":0, + "total_tokens":1012 + } + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IEmbeddingGenerator> generator = new ImageEmbeddingsClient( + new("https://somwhere"), new AzureKeyCredential("key"), new() + { + Transport = new HttpClientTransport(httpClient), + }).AsIEmbeddingGenerator("embed-v-4-0"); + + var response = await generator.GenerateAsync([dotnetPng]); + Assert.NotNull(response); + Assert.Single(response); + + Assert.NotNull(response.Usage); + Assert.Equal(1012, response.Usage.InputTokenCount); + Assert.Equal(1012, response.Usage.TotalTokenCount); + + foreach (Embedding e in response) + { + Assert.Equal("embed-v4.0", e.ModelId); + Assert.NotNull(e.CreatedAt); + Assert.Equal(1536, e.Vector.Length); + Assert.Contains(e.Vector.ToArray(), f => !f.Equals(0)); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj index d992413109b..a0f9abaf589 100644 --- a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj @@ -6,8 +6,17 @@ true + $(NoWarn);S104 + + + + + + + + @@ -18,5 +27,7 @@ + + diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs index 63b9897abbf..b56a2673b60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs @@ -18,6 +18,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Integration.Tests; +[Experimental("AIEVAL001")] public class QualityEvaluatorTests { private static readonly ChatOptions? _chatOptions; @@ -47,9 +48,7 @@ static QualityEvaluatorTests() string temperature = $"Temperature: {_chatOptions.Temperature}"; string usesContext = $"Feature: Context"; -#pragma warning disable AIEVAL001 IEvaluator rtcEvaluator = new RelevanceTruthAndCompletenessEvaluator(); -#pragma warning restore AIEVAL001 IEvaluator coherenceEvaluator = new CoherenceEvaluator(); IEvaluator fluencyEvaluator = new FluencyEvaluator(); @@ -72,7 +71,7 @@ static QualityEvaluatorTests() DiskBasedReportingConfiguration.Create( storageRootPath: Settings.Current.StorageRootPath, evaluators: [groundednessEvaluator, equivalenceEvaluator, completenessEvaluator, retrievalEvaluator], - chatConfiguration, + chatConfiguration: chatConfiguration, executionName: Constants.Version, tags: [version, date, projectName, testClass, provider, model, temperature, usesContext]); } @@ -101,6 +100,14 @@ await _qualityReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Warning), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(6, result.Metrics.Count); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.RelevanceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.TruthMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(CoherenceEvaluator.CoherenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(FluencyEvaluator.FluencyMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceEvaluator.RelevanceMetricName, out NumericMetric? _)); } [ConditionalFact] @@ -132,6 +139,14 @@ await _qualityReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity >= EvaluationDiagnosticSeverity.Warning), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(6, result.Metrics.Count); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.RelevanceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.TruthMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceTruthAndCompletenessEvaluator.CompletenessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(CoherenceEvaluator.CoherenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(FluencyEvaluator.FluencyMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(RelevanceEvaluator.RelevanceMetricName, out NumericMetric? _)); #if NET }); #else @@ -161,6 +176,17 @@ await _needsContextReportingConfiguration.CreateScenarioRunAsync( Assert.True( result.Metrics.Values.All(m => m.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error)), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(4, result.Metrics.Count); + Assert.True(result.TryGet(GroundednessEvaluator.GroundednessMetricName, out NumericMetric? groundedness)); + Assert.True(result.TryGet(EquivalenceEvaluator.EquivalenceMetricName, out NumericMetric? equivalence)); + Assert.True(result.TryGet(CompletenessEvaluator.CompletenessMetricName, out NumericMetric? completeness)); + Assert.True(result.TryGet(RetrievalEvaluator.RetrievalMetricName, out NumericMetric? retrieval)); + + Assert.Null(groundedness.Context); + Assert.Null(equivalence.Context); + Assert.Null(completeness.Context); + Assert.Null(retrieval.Context); } [ConditionalFact] @@ -224,6 +250,32 @@ await scenarioRun.EvaluateAsync( groundingContextForGroundednessEvaluator, groundTruthForCompletenessEvaluator, retrievedContextChunksForRetrievalEvaluator]); + + Assert.Equal(4, result.Metrics.Count); + Assert.True(result.TryGet(GroundednessEvaluator.GroundednessMetricName, out NumericMetric? groundedness)); + Assert.True(result.TryGet(EquivalenceEvaluator.EquivalenceMetricName, out NumericMetric? equivalence)); + Assert.True(result.TryGet(CompletenessEvaluator.CompletenessMetricName, out NumericMetric? completeness)); + Assert.True(result.TryGet(RetrievalEvaluator.RetrievalMetricName, out NumericMetric? retrieval)); + + Assert.True( + groundedness.Context?.Count is 1 && + groundedness.Context.TryGetValue(GroundednessEvaluatorContext.GroundingContextName, out EvaluationContext? context1) && + ReferenceEquals(context1, groundingContextForGroundednessEvaluator)); + + Assert.True( + equivalence.Context?.Count is 1 && + equivalence.Context.TryGetValue(EquivalenceEvaluatorContext.GroundTruthContextName, out EvaluationContext? context2) && + ReferenceEquals(context2, baselineResponseForEquivalenceEvaluator)); + + Assert.True( + completeness.Context?.Count is 1 && + completeness.Context.TryGetValue(CompletenessEvaluatorContext.GroundTruthContextName, out EvaluationContext? context3) && + ReferenceEquals(context3, groundTruthForCompletenessEvaluator)); + + Assert.True( + retrieval.Context?.Count is 1 && + retrieval.Context.TryGetValue(RetrievalEvaluatorContext.RetrievedContextChunksContextName, out EvaluationContext? context4) && + ReferenceEquals(context4, retrievedContextChunksForRetrievalEvaluator)); } [MemberNotNull(nameof(_qualityReportingConfiguration))] diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/README.md b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/README.md index bae6542da6a..e8ffbcc18e1 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/README.md +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/README.md @@ -1,7 +1,7 @@ ### Instructions -Some of the tests in this project (such as the tests in `EvaluatorTests.cs`) require special configuration to run. -These tests will be skipped by default if they have not been configured. +Some of the tests in this project (such as the tests in `QualityEvaluatorTests.cs` and `SafetyEvaluatorTests.cs`) +require special configuration to run. These tests will be skipped by default if they have not been configured. To configure the tests when running them locally on your machine, copy the `appsettings.json` file present in the current folder to a new file named `appsettings.local.json`, and fill in the values for the following properties: @@ -13,6 +13,9 @@ current folder to a new file named `appsettings.local.json`, and fill in the val "ModelName": "", "Endpoint": "", "StorageRootPath": "" + "AzureSubscriptionId": "", + "AzureResourceGroupName": "", + "AzureAIProjectName": "" } ``` diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs index 9a9ac055ec3..be6e08c1f43 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs @@ -153,17 +153,33 @@ At its furthest point (conjunction), Mars is about 250 million miles from Earth. The distance varies due to the elliptical orbits of both planets. """; - IEnumerable additionalContext = - [ - new GroundednessProEvaluatorContext(groundingContext), - new UngroundedAttributesEvaluatorContext(groundingContext) - ]; + GroundednessProEvaluatorContext groundednessProContext = new GroundednessProEvaluatorContext(groundingContext); + UngroundedAttributesEvaluatorContext ungroundedAttributesContext = new UngroundedAttributesEvaluatorContext(groundingContext); + IEnumerable additionalContext = [groundednessProContext, ungroundedAttributesContext]; EvaluationResult result = await scenarioRun.EvaluateAsync(messages, response, additionalContext); Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(6, result.Metrics.Count); + Assert.True(result.TryGet(SelfHarmEvaluator.SelfHarmMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(SexualEvaluator.SexualMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(GroundednessProEvaluator.GroundednessProMetricName, out NumericMetric? groundednessPro)); + Assert.True(result.TryGet(UngroundedAttributesEvaluator.UngroundedAttributesMetricName, out BooleanMetric? ungroundedAttributes)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); + + Assert.True( + groundednessPro.Context?.Count is 1 && + groundednessPro.Context.TryGetValue(GroundednessProEvaluatorContext.GroundingContextName, out EvaluationContext? context1) && + ReferenceEquals(context1, groundednessProContext)); + + Assert.True( + ungroundedAttributes.Context?.Count is 1 && + ungroundedAttributes.Context.TryGetValue(UngroundedAttributesEvaluatorContext.GroundingContextName, out EvaluationContext? context2) && + ReferenceEquals(context2, ungroundedAttributesContext)); } [ConditionalFact] @@ -212,17 +228,33 @@ At its closest (opposition), Jupiter is about 365 million miles away. At its furthest (conjunction), it can be approximately 601 million miles away. """; - IEnumerable additionalContext = - [ - new GroundednessProEvaluatorContext(groundingContext), - new UngroundedAttributesEvaluatorContext(groundingContext) - ]; + GroundednessProEvaluatorContext groundednessProContext = new GroundednessProEvaluatorContext(groundingContext); + UngroundedAttributesEvaluatorContext ungroundedAttributesContext = new UngroundedAttributesEvaluatorContext(groundingContext); + IEnumerable additionalContext = [groundednessProContext, ungroundedAttributesContext]; EvaluationResult result = await scenarioRun.EvaluateAsync(messages, response2, additionalContext); Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(6, result.Metrics.Count); + Assert.True(result.TryGet(SelfHarmEvaluator.SelfHarmMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(SexualEvaluator.SexualMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(GroundednessProEvaluator.GroundednessProMetricName, out NumericMetric? groundednessPro)); + Assert.True(result.TryGet(UngroundedAttributesEvaluator.UngroundedAttributesMetricName, out BooleanMetric? ungroundedAttributes)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); + + Assert.True( + groundednessPro.Context?.Count is 1 && + groundednessPro.Context.TryGetValue(GroundednessProEvaluatorContext.GroundingContextName, out EvaluationContext? context1) && + ReferenceEquals(context1, groundednessProContext)); + + Assert.True( + ungroundedAttributes.Context?.Count is 1 && + ungroundedAttributes.Context.TryGetValue(UngroundedAttributesEvaluatorContext.GroundingContextName, out EvaluationContext? context2) && + ReferenceEquals(context2, ungroundedAttributesContext)); } [ConditionalFact] @@ -250,6 +282,15 @@ await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(7, result.Metrics.Count); + Assert.True(result.TryGet(HateAndUnfairnessEvaluator.HateAndUnfairnessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ViolenceEvaluator.ViolenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedArtworkMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedFictionalCharactersMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedLogosAndBrandsMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -277,6 +318,15 @@ await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(7, result.Metrics.Count); + Assert.True(result.TryGet(HateAndUnfairnessEvaluator.HateAndUnfairnessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ViolenceEvaluator.ViolenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedArtworkMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedFictionalCharactersMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedLogosAndBrandsMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -317,6 +367,15 @@ await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(7, result.Metrics.Count); + Assert.True(result.TryGet(HateAndUnfairnessEvaluator.HateAndUnfairnessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ViolenceEvaluator.ViolenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedArtworkMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedFictionalCharactersMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedLogosAndBrandsMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -370,6 +429,15 @@ These distances are approximate and can vary slightly depending on the specific Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(7, result.Metrics.Count); + Assert.True(result.TryGet(HateAndUnfairnessEvaluator.HateAndUnfairnessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ViolenceEvaluator.ViolenceMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedMaterialMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedArtworkMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedFictionalCharactersMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(ProtectedMaterialEvaluator.ProtectedLogosAndBrandsMetricName, out BooleanMetric? _)); + Assert.True(result.TryGet(IndirectAttackEvaluator.IndirectAttackMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -396,6 +464,9 @@ await _codeVulnerabilityReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Single(result.Metrics); + Assert.True(result.TryGet(CodeVulnerabilityEvaluator.CodeVulnerabilityMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -434,6 +505,9 @@ await _codeVulnerabilityReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Single(result.Metrics); + Assert.True(result.TryGet(CodeVulnerabilityEvaluator.CodeVulnerabilityMetricName, out BooleanMetric? _)); } [ConditionalFact] @@ -465,6 +539,13 @@ await _mixedQualityAndSafetyReportingConfiguration.CreateScenarioRunAsync( Assert.False( result.ContainsDiagnostics(d => d.Severity is EvaluationDiagnosticSeverity.Error), string.Join("\r\n\r\n", result.Metrics.Values.SelectMany(m => m.Diagnostics ?? []).Select(d => d.ToString()))); + + Assert.Equal(5, result.Metrics.Count); + Assert.True(result.TryGet(FluencyEvaluator.FluencyMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(HateAndUnfairnessEvaluator.HateAndUnfairnessMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(SelfHarmEvaluator.SelfHarmMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(SexualEvaluator.SexualMetricName, out NumericMetric? _)); + Assert.True(result.TryGet(ViolenceEvaluator.ViolenceMetricName, out NumericMetric? _)); } [MemberNotNull(nameof(_contentSafetyReportingConfiguration))] diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs index b135a64a04c..2f936621147 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs @@ -47,9 +47,9 @@ public async Task DisposeAsync() internal override bool IsConfigured => Settings.Current.Configured; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new AzureStorageResponseCacheProvider(_dirClient!); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new AzureStorageResponseCacheProvider(_dirClient!, provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs index 610f6345524..62163d5e681 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs @@ -47,7 +47,7 @@ public async Task DisposeAsync() public override bool IsConfigured => Settings.Current.Configured; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new AzureStorageResultStore(_dirClient!); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs deleted file mode 100644 index b8351f45695..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; -using Xunit; -using CacheMode = Microsoft.Extensions.AI.Evaluation.Reporting.Storage.DiskBasedResponseCache.CacheMode; -using CacheOptions = Microsoft.Extensions.AI.Evaluation.Reporting.Storage.DiskBasedResponseCache.CacheOptions; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; - -public class CacheOptionsTests -{ - [Fact] - public void SerializeCacheOptions() - { - var options = new CacheOptions(CacheMode.Disabled, TimeSpan.FromDays(300)); - - string json = JsonSerializer.Serialize(options, JsonUtilities.Default.CacheOptionsTypeInfo); - CacheOptions? deserialized = JsonSerializer.Deserialize(json, JsonUtilities.Default.CacheOptionsTypeInfo); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized!.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - - } - - [Fact] - public void SerializeCacheOptionsCompact() - { - var options = new CacheOptions(CacheMode.Disabled, TimeSpan.FromDays(300)); - - string json = JsonSerializer.Serialize(options, JsonUtilities.Compact.CacheOptionsTypeInfo); - CacheOptions? deserialized = JsonSerializer.Deserialize(json, JsonUtilities.Default.CacheOptionsTypeInfo); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized!.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - - } - - [Fact] - public void SerializeCacheOptionsToFile() - { - var options = new CacheOptions(CacheMode.Enabled, TimeSpan.FromSeconds(10)); - - string tempFilePath = Path.GetTempFileName(); - options.Write(tempFilePath); - CacheOptions deserialized = CacheOptions.Read(tempFilePath); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - } - - [Fact] - public async Task SerializeCacheOptionsToFileAsync() - { - var options = new CacheOptions(CacheMode.Enabled, TimeSpan.FromSeconds(10)); - - string tempFilePath = Path.GetTempFileName(); - await options.WriteAsync(tempFilePath); - CacheOptions deserialized = await CacheOptions.ReadAsync(tempFilePath); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - } - -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs index e0ba0c171d1..8305fe8ddb3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs @@ -45,9 +45,9 @@ public Task DisposeAsync() internal override bool IsConfigured => true; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new DiskBasedResponseCacheProvider(UseTempStoragePath()); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new DiskBasedResponseCacheProvider(UseTempStoragePath(), provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs index 1fee1b9996c..77cabfd7ffd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs @@ -44,7 +44,7 @@ public Task DisposeAsync() public override bool IsConfigured => true; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new DiskBasedResultStore(UseTempStoragePath()); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs index 60e4e6f21ed..b69014e631b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; using Microsoft.Extensions.Caching.Distributed; using Microsoft.TestUtilities; using Xunit; @@ -19,8 +18,8 @@ public abstract class ResponseCacheTester private static readonly string _keyB = "B Key"; private static readonly byte[] _responseB = Encoding.UTF8.GetBytes("Content B"); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); internal abstract bool IsConfigured { get; } private void SkipIfNotConfigured() @@ -38,7 +37,7 @@ public async Task AddUncachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(AddUncachedEntry), iterationName); Assert.NotNull(cache); @@ -59,7 +58,7 @@ public async Task RemoveCachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -86,7 +85,7 @@ public async Task CacheEntryExpiration() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -96,7 +95,7 @@ public async Task CacheEntryExpiration() cache.Set(_keyB, _responseB); Assert.True(_responseB.SequenceEqual(cache.Get(_keyB) ?? [])); - now = DateTime.UtcNow + DiskBasedResponseCache.CacheOptions.Default.TimeToLiveForCacheEntries; + now = DateTime.UtcNow + Defaults.DefaultTimeToLiveForCacheEntries; Assert.Null(await cache.GetAsync(_keyA)); Assert.Null(cache.Get(_keyB)); @@ -107,7 +106,7 @@ public async Task MultipleCacheInstances() { SkipIfNotConfigured(); - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); Assert.NotNull(cache); IDistributedCache cache2 = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); @@ -134,7 +133,7 @@ public async Task DeleteExpiredEntries() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -144,7 +143,7 @@ public async Task DeleteExpiredEntries() cache.Set(_keyB, _responseB); Assert.True(_responseB.SequenceEqual(cache.Get(_keyB) ?? [])); - now = DateTime.UtcNow + DiskBasedResponseCache.CacheOptions.Default.TimeToLiveForCacheEntries; + now = DateTime.UtcNow + Defaults.DefaultTimeToLiveForCacheEntries; await provider.DeleteExpiredCacheEntriesAsync(); @@ -164,7 +163,7 @@ public async Task ResetCache() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs index 1ce033b3cd7..995b77a8c5e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; public abstract class ResultStoreTester { - public abstract IResultStore CreateResultStore(); + public abstract IEvaluationResultStore CreateResultStore(); public abstract bool IsConfigured { get; } @@ -39,7 +39,8 @@ private static ScenarioRunResult CreateTestResult(string scenarioName, string it private static string ScenarioName(int n) => $"Test.Scenario.{n}"; private static string IterationName(int n) => $"Iteration {n}"; - private static async Task> LoadResultsAsync(int n, IResultStore resultStore) + private static async Task> + LoadResultsAsync(int n, IEvaluationResultStore resultStore) { List<(string executionName, string scenarioName, string iterationName)> results = []; await foreach (string executionName in resultStore.GetLatestExecutionNamesAsync(n)) @@ -69,7 +70,7 @@ public async Task WriteAndReadResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string newExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -108,7 +109,7 @@ public async Task WriteAndReadHistoricalResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string firstExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -152,7 +153,7 @@ public async Task DeleteExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -176,7 +177,7 @@ public async Task DeleteSomeExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName0 = $"Test Execution {Path.GetRandomFileName()}"; @@ -211,7 +212,7 @@ public async Task DeleteScenarios() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -246,7 +247,7 @@ public async Task DeleteIterations() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs deleted file mode 100644 index f538d1476b0..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; - -namespace Microsoft.Extensions.AI; - -internal sealed class BinaryEmbedding : Embedding -{ - public BinaryEmbedding(ReadOnlyMemory bits) - { - Bits = bits; - } - - public ReadOnlyMemory Bits { get; } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index 22531e14e22..edb6c5dd14c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -229,16 +229,7 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Paramet }); Assert.Contains(secretNumber.ToString(), response.Text); - - // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response - if (response.Usage is { } finalUsage) - { - var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); - var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); - } + AssertUsageAgainstActivities(response, activities); } [ConditionalFact] @@ -306,16 +297,7 @@ public virtual async Task FunctionInvocation_OptionalParameter() }); Assert.Contains(secretNumber.ToString(), response.Text); - - // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response - if (response.Usage is { } finalUsage) - { - var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); - var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); - } + AssertUsageAgainstActivities(response, activities); } [ConditionalFact] @@ -347,15 +329,21 @@ public virtual async Task FunctionInvocation_NestedParameters() }); Assert.Contains((secretNumber + 19).ToString(), response.Text); + AssertUsageAgainstActivities(response, activities); + } + private static void AssertUsageAgainstActivities(ChatResponse response, List activities) + { // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response + // usage data across all calls to produce a single Usage value on the final response. + // The FunctionInvokingChatClient then itself creates a span that will also be tagged with a sum + // across all consituent calls, which means our final answer will be double. if (response.Usage is { } finalUsage) { var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); + Assert.Equal(totalInputTokens, finalUsage.InputTokenCount * 2); + Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount * 2); } } @@ -942,7 +930,7 @@ public virtual async Task GetResponseAsync_StructuredOutput_NonNative() var response = await captureOutputChatClient.GetResponseAsync(""" Supply an object to represent Jimbo Smith from Cardiff. - """, useJsonSchema: false); + """, useJsonSchemaResponseFormat: false); Assert.Equal("Jimbo Smith", response.Result.FullName); Assert.Contains("Cardiff", response.Result.HomeTown); diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs index 1188e899e4d..1504d0d2488 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +#if NET +using System.Collections; +#endif using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -148,7 +151,14 @@ public async Task Quantization_Binary_EmbeddingsCompareSuccessfully() { for (int j = 0; j < embeddings.Count; j++) { - distances[i, j] = TensorPrimitives.HammingBitDistance(embeddings[i].Bits.Span, embeddings[j].Bits.Span); + distances[i, j] = TensorPrimitives.HammingBitDistance(ToArray(embeddings[i].Vector), ToArray(embeddings[j].Vector)); + + static byte[] ToArray(BitArray array) + { + byte[] result = new byte[(array.Length + 7) / 8]; + array.CopyTo(result, 0); + return result; + } } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs index 3bf33988146..ea87408da38 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Linq; #if NET @@ -46,12 +47,12 @@ private static BinaryEmbedding QuantizeToBinary(Embedding embedding) { ReadOnlySpan vector = embedding.Vector.Span; - var result = new byte[(int)Math.Ceiling(vector.Length / 8.0)]; + var result = new BitArray(vector.Length); for (int i = 0; i < vector.Length; i++) { if (vector[i] > 0) { - result[i / 8] |= (byte)(1 << (i % 8)); + result[i / 8] = true; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs index 4fdc36b5280..9ba9c743166 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs @@ -8,6 +8,8 @@ using System.ComponentModel; using System.Linq; using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Azure.AI.OpenAI; using Microsoft.Extensions.Caching.Distributed; @@ -184,9 +186,6 @@ public async Task BasicRequestResponse_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -257,8 +256,6 @@ public async Task BasicRequestResponse_Streaming() Assert.Equal(createdAt, updates[i].CreatedAt); Assert.Equal("gpt-4o-mini-2024-07-18", updates[i].ModelId); Assert.Equal(ChatRole.Assistant, updates[i].Role); - Assert.NotNull(updates[i].AdditionalProperties); - Assert.Equal("fp_f85bea6784", updates[i].AdditionalProperties![nameof(ChatCompletion.SystemFingerprint)]); Assert.Equal(i == 10 ? 0 : 1, updates[i].Contents.Count); Assert.Equal(i < 10 ? null : ChatFinishReason.Stop, updates[i].FinishReason); } @@ -280,7 +277,361 @@ public async Task BasicRequestResponse_Streaming() } [Fact] - public async Task NonStronglyTypedOptions_AllSent() + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_completion_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"auto" + } + """; + + const string Output = """ + { + "id": "chatcmpl-123", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new() + { + FrequencyPenalty = 0.75f, + MaxOutputTokenCount = 10, + TopP = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Seed = 42, +#pragma warning restore OPENAI001 + ToolChoice = ChatToolChoice.CreateAutoChoice(), + ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat() + }; + openAIOptions.StopSequences.Add("hello"); + openAIOptions.Tools.Add(ToOpenAIChatTool(tool)); + return openAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_completion_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}},"additionalProperties":false}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}},"additionalProperties":false}}} + ], + "tool_choice":"auto", + "stream":true, + "stream_options":{"include_usage":true} + } + """; + + const string Output = """ + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new() + { + FrequencyPenalty = 0.75f, + MaxOutputTokenCount = 10, + TopP = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Seed = 42, +#pragma warning restore OPENAI001 + ToolChoice = ChatToolChoice.CreateAutoChoice(), + ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat() + }; + openAIOptions.StopSequences.Add("hello"); + openAIOptions.Tools.Add(ToOpenAIChatTool(tool)); + return openAIOptions; + }, + ModelId = null, // has no effect, you cannot change the model of an OpenAI's ChatClient. + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_completion_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none" + } + """; + + const string Output = """ + { + "id": "chatcmpl-123", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new(); + Assert.Null(openAIOptions.FrequencyPenalty); + Assert.Null(openAIOptions.MaxOutputTokenCount); + Assert.Null(openAIOptions.TopP); + Assert.Null(openAIOptions.PresencePenalty); + Assert.Null(openAIOptions.Temperature); +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Assert.Null(openAIOptions.Seed); +#pragma warning restore OPENAI001 + Assert.Empty(openAIOptions.StopSequences); + Assert.Empty(openAIOptions.Tools); + Assert.Null(openAIOptions.ToolChoice); + Assert.Null(openAIOptions.ResponseFormat); + return openAIOptions; + }, + ModelId = null, // has no effect, you cannot change the model of an OpenAI's ChatClient. + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_completion_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none", + "stream":true, + "stream_options":{"include_usage":true} + } + """; + + const string Output = """ + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new(); + Assert.Null(openAIOptions.FrequencyPenalty); + Assert.Null(openAIOptions.MaxOutputTokenCount); + Assert.Null(openAIOptions.TopP); + Assert.Null(openAIOptions.PresencePenalty); + Assert.Null(openAIOptions.Temperature); +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Assert.Null(openAIOptions.Seed); +#pragma warning restore OPENAI001 + Assert.Empty(openAIOptions.StopSequences); + Assert.Empty(openAIOptions.Tools); + Assert.Null(openAIOptions.ToolChoice); + Assert.Null(openAIOptions.ResponseFormat); + return openAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + /// Converts an Extensions function to an OpenAI chat tool. + private static ChatTool ToOpenAIChatTool(AIFunction aiFunction) + { + bool? strict = + aiFunction.AdditionalProperties.TryGetValue("strictJsonSchema", out object? strictObj) && + strictObj is bool strictValue ? + strictValue : null; + + // Map to an intermediate model so that redundant properties are skipped. + var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema)!; + var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool)); + return ChatTool.CreateFunctionTool(aiFunction.Name, aiFunction.Description, functionParameters, strict); + } + + /// Used to create the JSON payload for an OpenAI chat tool description. + private sealed class ChatToolJson + { + [JsonPropertyName("type")] + public string Type { get; set; } = "object"; + + [JsonPropertyName("required")] + public HashSet Required { get; set; } = []; + + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } = []; + + [JsonPropertyName("additionalProperties")] + public bool AdditionalProperties { get; set; } + } + + [Fact] + public async Task StronglyTypedOptions_AllSent() { const string Input = """ { @@ -320,17 +671,18 @@ public async Task NonStronglyTypedOptions_AllSent() Assert.NotNull(await client.GetResponseAsync("hello", new() { AllowMultipleToolCalls = false, - AdditionalProperties = new() + RawRepresentationFactory = (c) => { - ["StoredOutputEnabled"] = true, - ["Metadata"] = new Dictionary + var openAIOptions = new ChatCompletionOptions { - ["something"] = "else", - }, - ["LogitBiases"] = new Dictionary { { 12, 34 } }, - ["IncludeLogProbabilities"] = true, - ["TopLogProbabilityCount"] = 42, - ["EndUserId"] = "12345", + StoredOutputEnabled = true, + IncludeLogProbabilities = true, + TopLogProbabilityCount = 42, + EndUserId = "12345", + }; + openAIOptions.Metadata.Add("something", "else"); + openAIOptions.LogitBiases.Add(12, 34); + return openAIOptions; }, })); } @@ -446,9 +798,6 @@ public async Task MultipleMessages_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -546,9 +895,6 @@ public async Task MultiPartSystemMessage_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -647,9 +993,6 @@ public async Task EmptyAssistantMessage_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -767,9 +1110,6 @@ public async Task FunctionCallContent_NonStreaming() FunctionCallContent fcc = Assert.IsType(response.Messages.Single().Contents[0]); Assert.Equal("GetPersonAge", fcc.Name); AssertExtensions.EqualFunctionCallParameters(new Dictionary { ["personName"] = "Alice" }, fcc.Arguments); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -852,9 +1192,6 @@ public async Task UnavailableBuiltInFunctionCall_NonStreaming() Assert.Single(response.Messages.Single().Contents); TextContent fcc = Assert.IsType(response.Messages.Single().Contents[0]); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -946,8 +1283,6 @@ public async Task FunctionCallContent_Streaming() Assert.Equal(createdAt, updates[i].CreatedAt); Assert.Equal("gpt-4o-mini-2024-07-18", updates[i].ModelId); Assert.Equal(ChatRole.Assistant, updates[i].Role); - Assert.NotNull(updates[i].AdditionalProperties); - Assert.Equal("fp_f85bea6784", updates[i].AdditionalProperties![nameof(ChatCompletion.SystemFingerprint)]); Assert.Equal(i < 7 ? null : ChatFinishReason.ToolCalls, updates[i].FinishReason); } @@ -1111,9 +1446,6 @@ public async Task AssistantMessageWithBothToolsAndContent_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -1229,9 +1561,6 @@ private static async Task DataContentMessage_Image_AdditionalPropertyDetail_NonS { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_b705f0c291", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } private static IChatClient CreateChatClient(HttpClient httpClient, string modelId) => diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs index 557eecc3c29..aae985c4c4b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs @@ -139,7 +139,7 @@ public async Task SuccessUsage_NoJsonSchema() }; var chatHistory = new List { new(ChatRole.User, "Hello") }; - var response = await client.GetResponseAsync(chatHistory, useJsonSchema: false, serializerOptions: JsonContext2.Default.Options); + var response = await client.GetResponseAsync(chatHistory, useJsonSchemaResponseFormat: false, serializerOptions: JsonContext2.Default.Options); // The response contains the deserialized result and other response properties Assert.Equal(1, response.Result.Id); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs index 374e617adba..4f2427d133c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs @@ -32,10 +32,13 @@ public void Ctor_ExpectedDefaults() Assert.True(cachingClient.CoalesceStreamingUpdates); } - [Fact] - public async Task CachesSuccessResultsAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CachesSuccessResultsAsync(bool conversationIdSet) { // Arrange + ChatOptions options = new() { ConversationId = conversationIdSet ? "123" : null }; // Verify that all the expected properties will round-trip through the cache, // even if this involves serialization @@ -82,20 +85,20 @@ public async Task CachesSuccessResultsAsync() }; // Make the initial request and do a quick sanity check - var result1 = await outer.GetResponseAsync("some input"); + var result1 = await outer.GetResponseAsync("some input", options); Assert.Same(expectedResponse, result1); Assert.Equal(1, innerCallCount); // Act - var result2 = await outer.GetResponseAsync("some input"); + var result2 = await outer.GetResponseAsync("some input", options); // Assert - Assert.Equal(1, innerCallCount); + Assert.Equal(conversationIdSet ? 2 : 1, innerCallCount); AssertResponsesEqual(expectedResponse, result2); // Act/Assert 2: Cache misses do not return cached results - await outer.GetResponseAsync("some modified input"); - Assert.Equal(2, innerCallCount); + await outer.GetResponseAsync("some modified input", options); + Assert.Equal(conversationIdSet ? 3 : 2, innerCallCount); } [Fact] @@ -207,10 +210,13 @@ public async Task DoesNotCacheCanceledResultsAsync() Assert.Equal("A good result", result2.Text); } - [Fact] - public async Task StreamingCachesSuccessResultsAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamingCachesSuccessResultsAsync(bool conversationIdSet) { // Arrange + ChatOptions options = new() { ConversationId = conversationIdSet ? "123" : null }; // Verify that all the expected properties will round-trip through the cache, // even if this involves serialization @@ -255,20 +261,20 @@ public async Task StreamingCachesSuccessResultsAsync() }; // Make the initial request and do a quick sanity check - var result1 = outer.GetStreamingResponseAsync("some input"); + var result1 = outer.GetStreamingResponseAsync("some input", options); await AssertResponsesEqualAsync(actualUpdate, result1); Assert.Equal(1, innerCallCount); // Act - var result2 = outer.GetStreamingResponseAsync("some input"); + var result2 = outer.GetStreamingResponseAsync("some input", options); // Assert - Assert.Equal(1, innerCallCount); - await AssertResponsesEqualAsync(expectedCachedResponse, result2); + Assert.Equal(conversationIdSet ? 2 : 1, innerCallCount); + await AssertResponsesEqualAsync(conversationIdSet ? actualUpdate : expectedCachedResponse, result2); // Act/Assert 2: Cache misses do not return cached results - await ToListAsync(outer.GetStreamingResponseAsync("some modified input")); - Assert.Equal(2, innerCallCount); + await ToListAsync(outer.GetStreamingResponseAsync("some modified input", options)); + Assert.Equal(conversationIdSet ? 3 : 2, innerCallCount); } [Theory] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs index 4302348cc56..26554946dca 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs @@ -532,11 +532,11 @@ public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry) Func configure = b => b.Use(c => new FunctionInvokingChatClient(new OpenTelemetryChatClient(c, sourceName: sourceName))); - await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure)); + await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure), streaming: false); - await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure)); + await InvokeAsync(() => InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configure), streaming: true); - async Task InvokeAsync(Func work) + async Task InvokeAsync(Func work, bool streaming) { var activities = new List(); using TracerProvider? tracerProvider = enableTelemetry ? @@ -552,9 +552,9 @@ async Task InvokeAsync(Func work) { Assert.Collection(activities, activity => Assert.Equal("chat", activity.DisplayName), - activity => Assert.Equal("Func1", activity.DisplayName), + activity => Assert.Equal("execute_tool Func1", activity.DisplayName), activity => Assert.Equal("chat", activity.DisplayName), - activity => Assert.Equal(nameof(FunctionInvokingChatClient), activity.DisplayName)); + activity => Assert.Equal(streaming ? "FunctionInvokingChatClient.GetStreamingResponseAsync" : "FunctionInvokingChatClient.GetResponseAsync", activity.DisplayName)); for (int i = 0; i < activities.Count - 1; i++) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs index c6a4adb1e97..25e01afb1df 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Embeddings/OpenTelemetryEmbeddingGeneratorTests.cs @@ -16,9 +16,10 @@ namespace Microsoft.Extensions.AI; public class OpenTelemetryEmbeddingGeneratorTests { [Theory] - [InlineData(null)] - [InlineData("replacementmodel")] - public async Task ExpectedInformationLogged_Async(string? perRequestModelId) + [InlineData(null, false)] + [InlineData("replacementmodel", false)] + [InlineData("replacementmodel", true)] + public async Task ExpectedInformationLogged_Async(string? perRequestModelId, bool enableSensitiveData) { var sourceName = Guid.NewGuid().ToString(); var activities = new List(); @@ -45,7 +46,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) AdditionalProperties = new() { ["system_fingerprint"] = "abcdefgh", - ["AndSomethingElse"] = "value2", + ["AndSomethingElse"] = "value3", } }; }, @@ -56,7 +57,7 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) using var generator = innerGenerator .AsBuilder() - .UseOpenTelemetry(loggerFactory, sourceName) + .UseOpenTelemetry(loggerFactory, sourceName, configure: g => g.EnableSensitiveData = enableSensitiveData) .Build(); var options = new EmbeddingGenerationOptions @@ -85,12 +86,12 @@ public async Task ExpectedInformationLogged_Async(string? perRequestModelId) Assert.Equal(expectedModelName, activity.GetTagItem("gen_ai.request.model")); Assert.Equal(1234, activity.GetTagItem("gen_ai.request.embedding.dimensions")); - Assert.Equal("value1", activity.GetTagItem("gen_ai.testservice.request.service_tier")); - Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.request.something_else")); + Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("gen_ai.testservice.request.service_tier")); + Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("gen_ai.testservice.request.something_else")); Assert.Equal(10, activity.GetTagItem("gen_ai.response.input_tokens")); - Assert.Equal("abcdefgh", activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); - Assert.Equal("value2", activity.GetTagItem("gen_ai.testservice.response.and_something_else")); + Assert.Equal(enableSensitiveData ? "abcdefgh" : null, activity.GetTagItem("gen_ai.testservice.response.system_fingerprint")); + Assert.Equal(enableSensitiveData ? "value3" : null, activity.GetTagItem("gen_ai.testservice.response.and_something_else")); Assert.True(activity.Duration.TotalMilliseconds > 0); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs index 0670a06b206..4b5ff9a0600 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs @@ -809,6 +809,17 @@ public async Task MarshalResult_TypeIsDeclaredTypeEvenWhenDerivedTypeReturned() Assert.Equal("marshalResultInvoked", result); } + [Fact] + public async Task AIFunctionFactory_DefaultDefaultParameter() + { + Assert.NotEqual(new StructWithDefaultCtor().Value, default(StructWithDefaultCtor).Value); + + AIFunction f = AIFunctionFactory.Create((Guid g = default, StructWithDefaultCtor s = default) => g.ToString() + "," + s.Value.ToString(), serializerOptions: JsonContext.Default.Options); + + object? result = await f.InvokeAsync(); + Assert.Contains("00000000-0000-0000-0000-000000000000,0", result?.ToString()); + } + private sealed class MyService(int value) { public int Value => value; @@ -871,7 +882,19 @@ private class A; private class B : A; private sealed class C : B; + public readonly struct StructWithDefaultCtor + { + public int Value { get; } + public StructWithDefaultCtor() + { + Value = 42; + } + } + [JsonSerializable(typeof(IAsyncEnumerable))] [JsonSerializable(typeof(int[]))] + [JsonSerializable(typeof(string))] + [JsonSerializable(typeof(Guid))] + [JsonSerializable(typeof(StructWithDefaultCtor))] private partial class JsonContext : JsonSerializerContext; } diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs new file mode 100644 index 00000000000..1b8c4177f40 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebExecutionTests.cs @@ -0,0 +1,161 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Contains execution tests for the "AI Chat Web" template. +/// +/// +/// In addition to validating that the templates build and restore correctly, +/// these tests are also responsible for template component governance reporting. +/// This is because the generated output is left on disk after tests complete, +/// most importantly the project.assets.json file that gets created during restore. +/// Therefore, it's *critical* that these tests remain in a working state, +/// as disabling them will also disable CG reporting. +/// +public class AIChatWebExecutionTests : TemplateExecutionTestBase, ITemplateExecutionTestConfigurationProvider +{ + public AIChatWebExecutionTests(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper) + : base(fixture, outputHelper) + { + } + + public static TemplateExecutionTestConfiguration Configuration { get; } = new() + { + TemplatePackageName = "Microsoft.Extensions.AI.Templates", + TestOutputFolderPrefix = "AIChatWeb" + }; + + public static IEnumerable GetBasicTemplateOptions() + => GetFilteredTemplateOptions("--aspire", "false"); + + public static IEnumerable GetAspireTemplateOptions() + => GetFilteredTemplateOptions("--aspire", "true"); + + // Do not skip. See XML docs for this test class. + [Theory] + [MemberData(nameof(GetBasicTemplateOptions))] + public async Task CreateRestoreAndBuild_BasicTemplate(params string[] args) + { + const string ProjectName = "BasicApp"; + var project = await Fixture.CreateProjectAsync( + templateName: "aichatweb", + projectName: ProjectName, + args); + + await Fixture.RestoreProjectAsync(project); + await Fixture.BuildProjectAsync(project); + } + + // Do not skip. See XML docs for this test class. + [Theory] + [MemberData(nameof(GetAspireTemplateOptions))] + public async Task CreateRestoreAndBuild_AspireTemplate(params string[] args) + { + const string ProjectName = "AspireApp"; + var project = await Fixture.CreateProjectAsync( + templateName: "aichatweb", + ProjectName, + args); + + project.StartupProjectRelativePath = $"{ProjectName}.AppHost"; + + await Fixture.RestoreProjectAsync(project); + await Fixture.BuildProjectAsync(project); + } + + private static readonly (string name, string[] values)[] _templateOptions = [ + ("--provider", ["azureopenai", "githubmodels", "ollama", "openai"]), + ("--vector-store", ["azureaisearch", "local", "qdrant"]), + ("--managed-identity", ["true", "false"]), + ("--aspire", ["true", "false"]), + ]; + + private static IEnumerable GetFilteredTemplateOptions(params string[] filter) + { + foreach (var options in GetAllPossibleOptions(_templateOptions)) + { + if (!MatchesFilter()) + { + continue; + } + + if (HasOption("--managed-identity", "true")) + { + if (HasOption("--aspire", "true")) + { + // The managed identity option is disabled for the Aspire template. + continue; + } + + if (!HasOption("--vector-store", "azureaisearch") && + !HasOption("--aspire", "false")) + { + // Can only use managed identity when using Azure in the non-Aspire template. + continue; + } + } + + if (HasOption("--vector-store", "qdrant") && + HasOption("--aspire", "false")) + { + // Can't use Qdrant without Aspire. + continue; + } + + yield return options; + + bool MatchesFilter() + { + for (var i = 0; i < filter.Length; i += 2) + { + if (!HasOption(filter[i], filter[i + 1])) + { + return false; + } + } + + return true; + } + + bool HasOption(string name, string value) + { + for (var i = 0; i < options.Length; i += 2) + { + if (string.Equals(name, options[i], StringComparison.Ordinal) && + string.Equals(value, options[i + 1], StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } + } + } + + private static IEnumerable GetAllPossibleOptions(ReadOnlyMemory<(string name, string[] values)> options) + { + if (options.Length == 0) + { + yield return []; + yield break; + } + + var first = options.Span[0]; + foreach (var restSelection in GetAllPossibleOptions(options[1..])) + { + foreach (var value in first.values) + { + yield return [first.name, value, .. restSelection]; + } + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs similarity index 92% rename from test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs rename to test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs index db142ff68ff..1e4cf0415f4 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AichatwebTemplatesTests.cs +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/AIChatWebSnapshotTests.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Templates.IntegrationTests; using Microsoft.Extensions.AI.Templates.Tests; using Microsoft.Extensions.Logging; using Microsoft.TemplateEngine.Authoring.TemplateVerifier; @@ -14,9 +13,9 @@ using Xunit; using Xunit.Abstractions; -namespace Microsoft.Extensions.AI.Templates.InegrationTests; +namespace Microsoft.Extensions.AI.Templates.Tests; -public class AichatwebTemplatesTests : TestBase +public class AIChatWebSnapshotTests { // Keep the exclude patterns below in sync with those in Microsoft.Extensions.AI.Templates.csproj. private static readonly string[] _verificationExcludePatterns = [ @@ -36,7 +35,7 @@ public class AichatwebTemplatesTests : TestBase private readonly ILogger _log; - public AichatwebTemplatesTests(ITestOutputHelper log) + public AIChatWebSnapshotTests(ITestOutputHelper log) { #pragma warning disable CA2000 // Dispose objects before losing scope _log = new XunitLoggerProvider(log).CreateLogger("TestRun"); @@ -67,7 +66,7 @@ private async Task TestTemplateCoreAsync(string scenarioName, IEnumerable args) + { + FileName = WellKnownPaths.RepoDotNetExePath; + + foreach (var arg in args) + { + Arguments.Add(arg); + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs new file mode 100644 index 00000000000..cdd6ab73f03 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/DotNetNewCommand.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class DotNetNewCommand : DotNetCommand +{ + private bool _customHiveSpecified; + + public DotNetNewCommand(params ReadOnlySpan args) + : base(["new", .. args]) + { + } + + public DotNetNewCommand WithCustomHive(string path) + { + Arguments.Add("--debug:custom-hive"); + Arguments.Add(path); + _customHiveSpecified = true; + return this; + } + + public override Task ExecuteAsync(ITestOutputHelper outputHelper) + { + if (!_customHiveSpecified) + { + // If this exception starts getting thrown in cases where a custom hive is + // legitimately undesirable, we can add a new 'WithoutCustomHive()' method that + // just sets '_customHiveSpecified' to 'true'. + throw new InvalidOperationException($"A {nameof(DotNetNewCommand)} should specify a custom hive with '{nameof(WithCustomHive)}()'."); + } + + return base.ExecuteAsync(outputHelper); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs new file mode 100644 index 00000000000..3a499013495 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ITemplateExecutionTestConfigurationProvider.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public interface ITemplateExecutionTestConfigurationProvider +{ + static abstract TemplateExecutionTestConfiguration Configuration { get; } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs new file mode 100644 index 00000000000..d81c1f7c434 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/MessageSinkTestOutputHelper.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit.Abstractions; +using Xunit.Sdk; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class MessageSinkTestOutputHelper : ITestOutputHelper +{ + private readonly IMessageSink _messageSink; + + public MessageSinkTestOutputHelper(IMessageSink messageSink) + { + _messageSink = messageSink; + } + + public void WriteLine(string message) + { + _messageSink.OnMessage(new DiagnosticMessage(message)); + } + + public void WriteLine(string format, params object[] args) + { + _messageSink.OnMessage(new DiagnosticMessage(format, args)); + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs new file mode 100644 index 00000000000..a20d390794d --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/ProcessExtensions.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Diagnostics; + +public static class ProcessExtensions +{ + public static bool TryGetHasExited(this Process process) + { + try + { + return process.HasExited; + } + catch (InvalidOperationException ex) when (ex.Message.Contains("No process is associated with this object")) + { + return true; + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs new file mode 100644 index 00000000000..38ced5b1867 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/Project.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class Project(string rootPath, string name) +{ + private string? _startupProjectRelativePath; + private string? _startupProjectFullPath; + + public string RootPath => rootPath; + + public string Name => name; + + public string? StartupProjectRelativePath + { + get => _startupProjectRelativePath; + set + { + if (value is null) + { + _startupProjectRelativePath = null; + _startupProjectFullPath = null; + } + else if (!string.Equals(value, _startupProjectRelativePath, StringComparison.Ordinal)) + { + _startupProjectRelativePath = value; + _startupProjectFullPath = Path.Combine(rootPath, _startupProjectRelativePath); + } + } + } + + public string StartupProjectFullPath => _startupProjectFullPath ?? rootPath; +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs new file mode 100644 index 00000000000..b52e8cda3a6 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestBase.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Represents a test that executes a project template (create, restore, build, and run). +/// +/// A type defining global test execution settings. +[Collection(TemplateExecutionTestCollection.Name)] +public abstract class TemplateExecutionTestBase : IClassFixture.TemplateExecutionTestFixture>, IDisposable + where TConfiguration : ITemplateExecutionTestConfigurationProvider +{ + private bool _disposed; + + protected TemplateExecutionTestFixture Fixture { get; } + + protected ITestOutputHelper OutputHelper { get; } + + protected TemplateExecutionTestBase(TemplateExecutionTestFixture fixture, ITestOutputHelper outputHelper) + { + Fixture = fixture; + Fixture.SetCurrentTestOutputHelper(outputHelper); + + OutputHelper = outputHelper; + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) + { + return; + } + + _disposed = true; + + if (disposing) + { + Fixture.SetCurrentTestOutputHelper(null); + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// An implementation of that utilizes + /// the configuration provided by TConfiguration. + /// + /// + /// The configuration has to be provided "statically" because the lifetime of the class fixture + /// is longer than the lifetime of each test class instance. In other words, it's not possible for + /// an instance of the test class to configure to the fixture directly, as the test class instance + /// gets created after the fixture has a chance to perform global setup. + /// + /// The The . + public sealed class TemplateExecutionTestFixture(IMessageSink messageSink) + : TemplateExecutionTestClassFixtureBase(TConfiguration.Configuration, messageSink); +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs new file mode 100644 index 00000000000..3592fd6474b --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestClassFixtureBase.cs @@ -0,0 +1,129 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Provides functionality scoped to the duration of all the tests in a single test class +/// extending . +/// +public abstract class TemplateExecutionTestClassFixtureBase : IAsyncLifetime +{ + private readonly TemplateExecutionTestConfiguration _configuration; + private readonly string _templateTestOutputPath; + private readonly string _customHivePath; + private readonly MessageSinkTestOutputHelper _messageSinkTestOutputHelper; + private ITestOutputHelper? _currentTestOutputHelper; + + /// + /// Gets the current preferred output helper. + /// If a test is underway, the output will be associated with that test. + /// Otherwise, the output will appear as a diagnostic message via . + /// + private ITestOutputHelper OutputHelper => _currentTestOutputHelper ?? _messageSinkTestOutputHelper; + + protected TemplateExecutionTestClassFixtureBase(TemplateExecutionTestConfiguration configuration, IMessageSink messageSink) + { + _configuration = configuration; + _messageSinkTestOutputHelper = new(messageSink); + + var outputFolderName = GetRandomizedFileName(prefix: _configuration.TestOutputFolderPrefix); + _templateTestOutputPath = Path.Combine(WellKnownPaths.TemplateSandboxOutputRoot, outputFolderName); + _customHivePath = Path.Combine(_templateTestOutputPath, "hive"); + } + + private static string GetRandomizedFileName(string prefix) + => prefix + "_" + Guid.NewGuid().ToString("N").Substring(0, 10).ToLowerInvariant(); + + public async Task InitializeAsync() + { + Directory.CreateDirectory(_templateTestOutputPath); + + await InstallTemplatesAsync(); + + async Task InstallTemplatesAsync() + { + var installSandboxPath = Path.Combine(_templateTestOutputPath, "install"); + Directory.CreateDirectory(installSandboxPath); + + var installNuGetConfigPath = Path.Combine(installSandboxPath, "nuget.config"); + File.Copy(WellKnownPaths.TemplateInstallNuGetConfigPath, installNuGetConfigPath); + + var installResult = await new DotNetNewCommand("install", _configuration.TemplatePackageName) + .WithWorkingDirectory(installSandboxPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", WellKnownPaths.LocalShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", WellKnownPaths.NuGetPackagesPath) + .WithCustomHive(_customHivePath) + .ExecuteAsync(OutputHelper); + installResult.AssertSucceeded(); + } + } + + public async Task CreateProjectAsync(string templateName, string projectName, params string[] args) + { + var outputFolderName = GetRandomizedFileName(projectName); + var outputFolderPath = Path.Combine(_templateTestOutputPath, outputFolderName); + + ReadOnlySpan dotNetNewCommandArgs = [ + templateName, + "-o", outputFolderPath, + "-n", projectName, + "--no-update-check", + .. args + ]; + + var newProjectResult = await new DotNetNewCommand(dotNetNewCommandArgs) + .WithWorkingDirectory(_templateTestOutputPath) + .WithCustomHive(_customHivePath) + .ExecuteAsync(OutputHelper); + newProjectResult.AssertSucceeded(); + + var templateNuGetConfigPath = Path.Combine(outputFolderPath, "nuget.config"); + File.Copy(WellKnownPaths.TemplateTestNuGetConfigPath, templateNuGetConfigPath); + + return new Project(outputFolderPath, projectName); + } + + public async Task RestoreProjectAsync(Project project) + { + var restoreResult = await new DotNetCommand("restore") + .WithWorkingDirectory(project.StartupProjectFullPath) + .WithEnvironmentVariable("LOCAL_SHIPPING_PATH", WellKnownPaths.LocalShippingPackagesPath) + .WithEnvironmentVariable("NUGET_PACKAGES", WellKnownPaths.NuGetPackagesPath) + .ExecuteAsync(OutputHelper); + restoreResult.AssertSucceeded(); + } + + public async Task BuildProjectAsync(Project project) + { + var buildResult = await new DotNetCommand("build", "--no-restore") + .WithWorkingDirectory(project.StartupProjectFullPath) + .ExecuteAsync(OutputHelper); + buildResult.AssertSucceeded(); + } + + public void SetCurrentTestOutputHelper(ITestOutputHelper? outputHelper) + { + if (_currentTestOutputHelper is not null && outputHelper is not null) + { + throw new InvalidOperationException( + "Cannot set the template execution test output helper when one is already present. " + + "This might be a sign that template execution tests are running in parallel, " + + "which is not currently supported."); + } + + _currentTestOutputHelper = outputHelper; + } + + public Task DisposeAsync() + { + // Only here to implement IAsyncLifetime. Not currently used. + return Task.CompletedTask; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs new file mode 100644 index 00000000000..9f10ffdf974 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollection.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +[CollectionDefinition(name: Name)] +public sealed class TemplateExecutionTestCollection : ICollectionFixture +{ + public const string Name = "Template execution test"; +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs new file mode 100644 index 00000000000..13140b0599e --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestCollectionFixture.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Provides functionality scoped to the lifetime of all tests defined in +/// test classes extending . +/// +public sealed class TemplateExecutionTestCollectionFixture +{ + public TemplateExecutionTestCollectionFixture() + { + // Here, we clear execution test output from the previous test run, if it exists. + // + // It's critical that this clearing happens *before* the tests start, *not* after they complete. + // + // This is because: + // 1. This enables debugging the previous test run by building/running generated projects manually. + // 2. The existence of a project.assets.json file on disk is what allows template content to get discovered + // for component governance reporting. + if (Directory.Exists(WellKnownPaths.TemplateSandboxOutputRoot)) + { + Directory.Delete(WellKnownPaths.TemplateSandboxOutputRoot, recursive: true); + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs new file mode 100644 index 00000000000..ce621e58528 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TemplateExecutionTestConfiguration.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TemplateExecutionTestConfiguration +{ + public required string TemplatePackageName { get; init; } + + public required string TestOutputFolderPrefix { get; init; } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs new file mode 100644 index 00000000000..697bf009f9c --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommand.cs @@ -0,0 +1,125 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Xunit.Abstractions; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public abstract class TestCommand +{ + public string? FileName { get; set; } + + public string? WorkingDirectory { get; set; } + + public TimeSpan? Timeout { get; set; } + + public List Arguments { get; } = []; + + public Dictionary EnvironmentVariables = []; + + public virtual async Task ExecuteAsync(ITestOutputHelper outputHelper) + { + if (string.IsNullOrEmpty(FileName)) + { + throw new InvalidOperationException($"The {nameof(TestCommand)} did not specify an executable file name."); + } + + var processStartInfo = new ProcessStartInfo(FileName, Arguments) + { + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + UseShellExecute = false, + }; + + if (WorkingDirectory is not null) + { + processStartInfo.WorkingDirectory = WorkingDirectory; + } + + foreach (var (key, value) in EnvironmentVariables) + { + processStartInfo.EnvironmentVariables[key] = value; + } + + var exitedTcs = new TaskCompletionSource(); + var standardOutputBuilder = new StringBuilder(); + var standardErrorBuilder = new StringBuilder(); + + using var process = new Process + { + StartInfo = processStartInfo, + }; + + process.EnableRaisingEvents = true; + process.OutputDataReceived += MakeOnDataReceivedHandler(standardOutputBuilder); + process.ErrorDataReceived += MakeOnDataReceivedHandler(standardErrorBuilder); + process.Exited += (sender, args) => + { + exitedTcs.SetResult(); + }; + + DataReceivedEventHandler MakeOnDataReceivedHandler(StringBuilder outputBuilder) => (sender, args) => + { + if (args.Data is null) + { + return; + } + + lock (outputBuilder) + { + outputBuilder.AppendLine(args.Data); + } + + lock (outputHelper) + { + outputHelper.WriteLine(args.Data); + } + }; + + outputHelper.WriteLine($"Executing '{processStartInfo.FileName} {string.Join(" ", Arguments)}' in working directory '{processStartInfo.WorkingDirectory}'"); + + using var timeoutCts = new CancellationTokenSource(); + if (Timeout is { } timeout) + { + timeoutCts.CancelAfter(timeout); + } + + var startTimestamp = Stopwatch.GetTimestamp(); + + try + { + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await exitedTcs.Task.WaitAsync(timeoutCts.Token).ConfigureAwait(false); + await process.WaitForExitAsync(timeoutCts.Token).ConfigureAwait(false); + + var elapsedTime = Stopwatch.GetElapsedTime(startTimestamp); + outputHelper.WriteLine($"Process ran for {elapsedTime} seconds."); + + return new(standardOutputBuilder, standardErrorBuilder, process.ExitCode); + } + catch (Exception ex) + { + outputHelper.WriteLine($"An exception occurred: {ex}"); + throw; + } + finally + { + if (!process.TryGetHasExited()) + { + var elapsedTime = Stopwatch.GetElapsedTime(startTimestamp); + outputHelper.WriteLine($"The process has been running for {elapsedTime} seconds. Terminating the process."); + process.Kill(entireProcessTree: true); + } + } + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs new file mode 100644 index 00000000000..957c0efbb79 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandExtensions.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public static class TestCommandExtensions +{ + public static TCommand WithEnvironmentVariable(this TCommand command, string name, string value) + where TCommand : TestCommand + { + command.EnvironmentVariables[name] = value; + return command; + } + + public static TCommand WithWorkingDirectory(this TCommand command, string workingDirectory) + where TCommand : TestCommand + { + command.WorkingDirectory = workingDirectory; + return command; + } + + public static TCommand WithTimeout(this TCommand command, TimeSpan timeout) + where TCommand : TestCommand + { + command.Timeout = timeout; + return command; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs new file mode 100644 index 00000000000..09d09d50a1c --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResult.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public sealed class TestCommandResult(StringBuilder standardOutputBuilder, StringBuilder standardErrorBuilder, int exitCode) +{ + private string? _standardOutput; + private string? _standardError; + + public string StandardOutput => _standardOutput ??= standardOutputBuilder.ToString(); + + public string StandardError => _standardError ??= standardErrorBuilder.ToString(); + + public int ExitCode => exitCode; +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs new file mode 100644 index 00000000000..867cc2303ac --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/TestCommandResultExtensions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +public static class TestCommandResultExtensions +{ + public static TestCommandResult AssertZeroExitCode(this TestCommandResult result) + { + Assert.True(result.ExitCode == 0, $"Expected an exit code of zero, got {result.ExitCode}"); + return result; + } + + public static TestCommandResult AssertEmptyStandardError(this TestCommandResult result) + { + var standardError = result.StandardError; + Assert.True(string.IsNullOrWhiteSpace(standardError), $"Standard error output was unexpectedly non-empty:\n{standardError}"); + return result; + } + + public static TestCommandResult AssertSucceeded(this TestCommandResult result) + => result + .AssertZeroExitCode() + .AssertEmptyStandardError(); +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs new file mode 100644 index 00000000000..0d399dfcfe7 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Infrastructure/WellKnownPaths.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.InteropServices; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +internal static class WellKnownPaths +{ + public static readonly string RepoRoot; + public static readonly string RepoDotNetExePath; + public static readonly string ThisProjectRoot; + + public static readonly string TemplateFeedLocation; + public static readonly string TemplateSandboxRoot; + public static readonly string TemplateSandboxOutputRoot; + public static readonly string TemplateInstallNuGetConfigPath; + public static readonly string TemplateTestNuGetConfigPath; + public static readonly string LocalShippingPackagesPath; + public static readonly string NuGetPackagesPath; + + static WellKnownPaths() + { + RepoRoot = GetRepoRoot(); + RepoDotNetExePath = GetRepoDotNetExePath(); + ThisProjectRoot = ProjectRootHelper.GetThisProjectRoot(); + + TemplateFeedLocation = Path.Combine(RepoRoot, "src", "ProjectTemplates"); + TemplateSandboxRoot = Path.Combine(ThisProjectRoot, "TemplateSandbox"); + TemplateSandboxOutputRoot = Path.Combine(TemplateSandboxRoot, "output"); + TemplateInstallNuGetConfigPath = Path.Combine(TemplateSandboxRoot, "nuget.template_install.config"); + TemplateTestNuGetConfigPath = Path.Combine(TemplateSandboxRoot, "nuget.template_test.config"); + + const string BuildConfigurationFolder = +#if DEBUG + "Debug"; +#else + "Release"; +#endif + LocalShippingPackagesPath = Path.Combine(RepoRoot, "artifacts", "packages", BuildConfigurationFolder, "Shipping"); + NuGetPackagesPath = Path.Combine(TemplateSandboxOutputRoot, "packages"); + } + + private static string GetRepoRoot() + { + string? directory = AppContext.BaseDirectory; + + while (directory is not null) + { + var gitPath = Path.Combine(directory, ".git"); + if (Directory.Exists(gitPath) || File.Exists(gitPath)) + { + // Found the repo root, which should either have a .git folder or, if the repo + // is part of a Git worktree, a .git file. + return directory; + } + + directory = Directory.GetParent(directory)?.FullName; + } + + throw new InvalidOperationException("Failed to establish root of the repository"); + } + + private static string GetRepoDotNetExePath() + { + var dotNetExeName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? "dotnet.exe" + : "dotnet"; + + var dotNetExePath = Path.Combine(RepoRoot, ".dotnet", dotNetExeName); + + if (!File.Exists(dotNetExePath)) + { + throw new InvalidOperationException($"Expected to find '{dotNetExeName}' at '{dotNetExePath}', but it was not found."); + } + + return dotNetExePath; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj index 2c1c66e4d3e..d2fc26ea0ab 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Microsoft.Extensions.AI.Templates.Tests.csproj @@ -1,7 +1,9 @@  - Unit tests for Microsoft.Extensions.AI.Templates. + Tests for Microsoft.Extensions.AI.Templates. + false + true @@ -16,7 +18,11 @@ + + + + + - diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs new file mode 100644 index 00000000000..3d076a438ad --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/ProjectRootHelper.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Runtime.CompilerServices; + +namespace Microsoft.Extensions.AI.Templates.Tests; + +/// +/// Contains a helper for determining the disk location of the containing project folder. +/// +/// +/// It's important that this file resides in the root of the containing project, or the returned +/// project root path will be incorrect. +/// +internal static class ProjectRootHelper +{ + public static string GetThisProjectRoot() + => GetThisProjectRootCore(); + + // This helper method is defined separately from its public variant because it extracts the + // caller file path via the [CallerFilePath] attribute. + // Therefore, the caller must be in a known location, i.e., this source file, to produce + // a reliable result. + private static string GetThisProjectRootCore([CallerFilePath] string callerFilePath = "") + { + if (Path.GetDirectoryName(callerFilePath) is not { Length: > 0 } testProjectRoot) + { + throw new InvalidOperationException("Could not determine the root of the test project."); + } + + return testProjectRoot; + } +} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md new file mode 100644 index 00000000000..ca589c22c2c --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/README.md @@ -0,0 +1,5 @@ +# Microsoft.Extensions.AI.Templates tests + +Contains snapshot and execution tests for `Microsoft.Extensions.AI.Templates`. + +For information on debugging template execution tests, see [this README](./TemplateSandbox/README.md). diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/dompurify/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/dompurify/README.md index 3a791cd9324..cc6e6b4153f 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/dompurify/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/dompurify/README.md @@ -2,4 +2,4 @@ https://github.com/cure53/DOMPurify License: Apache 2.0 and Mozilla Public License 2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/dompurify +To update, replace the `dist/purify.es.mjs` file with an updated version from https://www.npmjs.com/package/dompurify. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/marked/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/marked/README.md index 889562dd0ca..352b52d5503 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/marked/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/marked/README.md @@ -2,4 +2,4 @@ https://github.com/markedjs/marked License: MIT -To update, replace the files with with an updated build from https://www.npmjs.com/package/marked +To update, replace the `dist/marked.esm.js` file with with an updated version from https://www.npmjs.com/package/marked. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md index 4f4e041bcdf..8e77fba7d43 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md @@ -2,4 +2,9 @@ https://github.com/mozilla/pdf.js License: Apache-2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/pdfjs-dist +To update, replace the following files with updated versions from https://www.npmjs.com/package/pdfjs-dist: +* `build/pdf.min.mjs` +* `build/pdf.worker.min.mjs` +* `web/pdf_viewer.css` +* `web/pdf_viewer.mjs` +* `web/images/loading-icon.gif` diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md index fb11ec69863..c05c18281ef 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md @@ -47,6 +47,10 @@ Several .NET Aspire templates include ASP.NET Core projects that are configured See [Troubleshoot untrusted localhost certificate in .NET Aspire](https://learn.microsoft.com/dotnet/aspire/troubleshooting/untrusted-localhost-certificate) for more information. +# Updating JavaScript dependencies + +This template leverages JavaScript libraries to provide essential functionality. These libraries are located in the wwwroot/lib folder of the aichatweb.Web project. For instructions on updating each dependency, please refer to the README.md file in each respective folder. + # Learn More To learn more about development with .NET and AI, check out the following links: diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/dompurify/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/dompurify/README.md index 3a791cd9324..cc6e6b4153f 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/dompurify/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/dompurify/README.md @@ -2,4 +2,4 @@ https://github.com/cure53/DOMPurify License: Apache 2.0 and Mozilla Public License 2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/dompurify +To update, replace the `dist/purify.es.mjs` file with an updated version from https://www.npmjs.com/package/dompurify. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/marked/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/marked/README.md index 889562dd0ca..352b52d5503 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/marked/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/marked/README.md @@ -2,4 +2,4 @@ https://github.com/markedjs/marked License: MIT -To update, replace the files with with an updated build from https://www.npmjs.com/package/marked +To update, replace the `dist/marked.esm.js` file with with an updated version from https://www.npmjs.com/package/marked. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/pdfjs-dist/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/pdfjs-dist/README.md index 4f4e041bcdf..8e77fba7d43 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/pdfjs-dist/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/wwwroot/lib/pdfjs-dist/README.md @@ -2,4 +2,9 @@ https://github.com/mozilla/pdf.js License: Apache-2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/pdfjs-dist +To update, replace the following files with updated versions from https://www.npmjs.com/package/pdfjs-dist: +* `build/pdf.min.mjs` +* `build/pdf.worker.min.mjs` +* `web/pdf_viewer.css` +* `web/pdf_viewer.mjs` +* `web/images/loading-icon.gif` diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/README.md index f4a928c401a..53730a18884 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/README.md @@ -57,6 +57,10 @@ Make sure to replace `YOUR-AZURE-AI-SEARCH-ENDPOINT` with your actual Azure AI S 3. Once installed, Open the `Program.cs` file. 4. Run the project by clicking the "Run" button in the Debug view. +# Updating JavaScript dependencies + +This template leverages JavaScript libraries to provide essential functionality. These libraries are located in the wwwroot/lib folder. For instructions on updating each dependency, please refer to the README.md file in each respective folder. + # Learn More To learn more about development with .NET and AI, check out the following links: diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/dompurify/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/dompurify/README.md index 3a791cd9324..cc6e6b4153f 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/dompurify/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/dompurify/README.md @@ -2,4 +2,4 @@ https://github.com/cure53/DOMPurify License: Apache 2.0 and Mozilla Public License 2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/dompurify +To update, replace the `dist/purify.es.mjs` file with an updated version from https://www.npmjs.com/package/dompurify. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/marked/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/marked/README.md index 889562dd0ca..352b52d5503 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/marked/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/marked/README.md @@ -2,4 +2,4 @@ https://github.com/markedjs/marked License: MIT -To update, replace the files with with an updated build from https://www.npmjs.com/package/marked +To update, replace the `dist/marked.esm.js` file with with an updated version from https://www.npmjs.com/package/marked. diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md index 4f4e041bcdf..8e77fba7d43 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/wwwroot/lib/pdfjs-dist/README.md @@ -2,4 +2,9 @@ https://github.com/mozilla/pdf.js License: Apache-2.0 -To update, replace the files with an updated build from https://www.npmjs.com/package/pdfjs-dist +To update, replace the following files with updated versions from https://www.npmjs.com/package/pdfjs-dist: +* `build/pdf.min.mjs` +* `build/pdf.worker.min.mjs` +* `web/pdf_viewer.css` +* `web/pdf_viewer.mjs` +* `web/images/loading-icon.gif` diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig new file mode 100644 index 00000000000..b8f20c69836 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.editorconfig @@ -0,0 +1,2 @@ +# Don't apply the repo's editorconfig settings to generated templates. +root = true diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore new file mode 100644 index 00000000000..ee80e74117d --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/.gitignore @@ -0,0 +1,2 @@ +# Template test output +output/ diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props new file mode 100644 index 00000000000..e3b34086f94 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.props @@ -0,0 +1,11 @@ + + + + + + true + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets new file mode 100644 index 00000000000..ecef22f1080 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/Directory.Build.targets @@ -0,0 +1,6 @@ + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md new file mode 100644 index 00000000000..7db29aaab19 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/README.md @@ -0,0 +1,31 @@ +# Template test sandbox + +This folder exists to serve as an isolated environment for template execution tests. + +## Debugging template execution tests + +Before running template execution tests, make sure that packages defined in this solution have been packed by running the following commands from the repo root: +```sh +./build.cmd -build +./build.cmd -pack +``` + +**Note:** These commands currently need to be run separately so that generated template content gets included in the template `.nupkg`. + +Template tests can be debugged either in VS or by running `dotnet test`. + +However, it's sometimes helpful to debug failures by building, running, and modifying the generated projects directly instead of tinkering with test code. + +To help with this scenario: +* The `output/` folder containing the generated projects doesn't get cleared until the start of the next test run. +* An `activate.ps1` script can be used to simulate the environment that the template execution tests use. This script: + * Sets the active .NET installation to `/.dotnet`. + * Sets the `NUGET_PACKAGES` environment variable to the `output/packages` folder to use the isolated package cache. + * Sets a `LOCAL_SHIPPING_PATH` environment variable so that locally-built packages can get picked up during restore. + +As an example, here's how you can build a project generated by the tests: +```sh +. ./activate.ps1 +cd ./output/[test_collection]/[generated_template] +dotnet build +``` diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 new file mode 100644 index 00000000000..a3dea765dd5 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/activate.ps1 @@ -0,0 +1,109 @@ +# +# This file creates an environment similar to the one that the template tests use. +# This makes it convenient to restore, build, and run projects generated by the template tests +# to debug test failures. +# +# This file must be used by invoking ". .\activate.ps1" from the command line. +# You cannot run it directly. See https://learn.microsoft.com/powershell/module/microsoft.powershell.core/about/about_scripts#script-scope-and-dot-sourcing +# +# To exit from the environment this creates, execute the 'deactivate' function. +# + +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('c')]$configuration = "Debug" +) + +if ($MyInvocation.CommandOrigin -eq 'runspace') { + $cwd = (Get-Location).Path + $scriptPath = $MyInvocation.MyCommand.Path + $relativePath = [System.IO.Path]::GetRelativePath($cwd, $scriptPath) + Write-Host -f Red "This script cannot be invoked directly." + Write-Host -f Red "To function correctly, this script file must be 'dot sourced' by calling `". .\$relativePath`" (notice the dot at the beginning)." + exit 1 +} + +function deactivate ([switch]$init) { + # reset old environment variables + if (Test-Path variable:_OLD_PATH) { + $env:PATH = $_OLD_PATH + Remove-Item variable:_OLD_PATH + } + + if (test-path function:_old_prompt) { + Set-Item Function:prompt -Value $function:_old_prompt -ea ignore + remove-item function:_old_prompt + } + + Remove-Item env:DOTNET_ROOT -ea Ignore + Remove-Item 'env:DOTNET_ROOT(x86)' -ea Ignore + Remove-Item env:DOTNET_MULTILEVEL_LOOKUP -ea Ignore + Remove-Item env:NUGET_PACKAGES -ea Ignore + Remove-Item env:LOCAL_SHIPPING_PATH -ea Ignore + if (-not $init) { + # Remove functions defined + Remove-Item function:deactivate + Remove-Item function:Get-RepoRoot + } +} + +# Cleanup the environment +deactivate -init + +function Get-RepoRoot { + $directory = $PSScriptRoot + + while ($directory) { + $gitPath = Join-Path $directory ".git" + + if (Test-Path $gitPath) { + return $directory + } + + $parent = Split-Path $directory -Parent + if ($parent -eq $directory) { + # We've reached the filesystem root + break + } + + $directory = $parent + } + + throw "Failed to establish root of the repository" +} + +# Find the root of the repository +$repoRoot = Get-RepoRoot + +$_OLD_PATH = $env:PATH +# Tell dotnet where to find itself +$env:DOTNET_ROOT = "$repoRoot\.dotnet" +${env:DOTNET_ROOT(x86)} = "$repoRoot\.dotnet\x86" +# Tell dotnet not to look beyond the DOTNET_ROOT folder for more dotnet things +$env:DOTNET_MULTILEVEL_LOOKUP = 0 +# Put dotnet first on PATH +$env:PATH = "${env:DOTNET_ROOT};${env:PATH}" +# Set NUGET_PACKAGES and LOCAL_SHIPPING_PATH +$env:NUGET_PACKAGES = "$PSScriptRoot\output\packages" +$env:LOCAL_SHIPPING_PATH = "$repoRoot\artifacts\packages\$configuration\Shipping\" + +# Set the shell prompt +$function:_old_prompt = $function:prompt +function dotnet_prompt { + # Add a prefix to the current prompt, but don't discard it. + write-host "($( split-path $PSScriptRoot -leaf )) " -nonewline + & $function:_old_prompt +} + +Set-Item Function:prompt -Value $function:dotnet_prompt -ea ignore + +Write-Host -f Magenta "Enabled the template testing environment. Execute 'deactivate' to exit." +Write-Host -f Magenta "Using the '$configuration' configuration. Use the -c option to specify a different configuration." +if (-not (Test-Path "${env:DOTNET_ROOT}\dotnet.exe")) { + Write-Host -f Yellow ".NET Core has not been installed yet. Run $repoRoot\build.cmd -restore to install it." +} +else { + Write-Host "dotnet = ${env:DOTNET_ROOT}\dotnet.exe" +} +Write-Host "NUGET_PACKAGES = ${env:NUGET_PACKAGES}" +Write-Host "LOCAL_SHIPPING_PATH = ${env:LOCAL_SHIPPING_PATH}" diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config new file mode 100644 index 00000000000..4d77494baee --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_install.config @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config new file mode 100644 index 00000000000..c98b9777011 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TemplateSandbox/nuget.template_test.config @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs deleted file mode 100644 index 59946ab283f..00000000000 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/TestBase.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; - -namespace Microsoft.Extensions.AI.Templates.IntegrationTests; - -/// -/// The class contains the utils for unit and integration tests. -/// -public abstract class TestBase -{ - internal static string CodeBaseRoot { get; } = GetCodeBaseRoot(); - - internal static string TemplateFeedLocation { get; } = Path.Combine(CodeBaseRoot, "src", "ProjectTemplates"); - - private static string GetCodeBaseRoot() - { - string? directory = AppContext.BaseDirectory; - - while (directory is not null) - { - var gitPath = Path.Combine(directory, ".git"); - if (Directory.Exists(gitPath) || File.Exists(gitPath)) - { - // Found the repo root, which should either have a .git folder or, if the repo - // is part of a Git worktree, a .git file. - return directory; - } - - directory = Directory.GetParent(directory)?.FullName; - } - - throw new InvalidOperationException("Failed to establish root of the repository"); - } -} diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json new file mode 100644 index 00000000000..9faab404ec0 --- /dev/null +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/xunit.runner.json @@ -0,0 +1,5 @@ +{ + "longRunningTestSeconds": 60, + "diagnosticMessages": true, + "maxParallelThreads": 1 +}