Remove ImageContent and AudioContent (#5814)

Author: MackinnonBuck

eng/MSBuild/Shared.props

@@ -14,7 +14,7 @@
   <ItemGroup Condition="'$(InjectSharedRentedSpan)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\RentedSpan\*.cs" LinkBase="Shared\RentedSpan" />
   </ItemGroup>
-  
+
   <ItemGroup Condition="'$(InjectSharedServerSentEvents)' == 'true'">
     <Compile Include="$(MSBuildThisFileDirectory)\..\..\src\Shared\ServerSentEvents\*.cs" LinkBase="Shared\ServerSentEvents" />
   </ItemGroup>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs

@@ -7,11 +7,9 @@ namespace Microsoft.Extensions.AI;
 
 /// <summary>Provides a base class for all content used with AI services.</summary>
 [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
-[JsonDerivedType(typeof(AudioContent), typeDiscriminator: "audio")]
 [JsonDerivedType(typeof(DataContent), typeDiscriminator: "data")]
 [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: "functionCall")]
 [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: "functionResult")]
-[JsonDerivedType(typeof(ImageContent), typeDiscriminator: "image")]
 [JsonDerivedType(typeof(TextContent), typeDiscriminator: "text")]
 [JsonDerivedType(typeof(UsageContent), typeDiscriminator: "usage")]
 public class AIContent

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AudioContent.cs

@@ -105,6 +105,14 @@ public DataContent(ReadOnlyMemory<byte> data, string? mediaType = null)
         _data = data;
     }
 
+    /// <summary>
+    /// Determines whether the <see cref="MediaType"/> has the specified prefix.
+    /// </summary>
+    /// <param name="prefix">The media type prefix.</param>
+    /// <returns><see langword="true"/> if the <see cref="MediaType"/> has the specified prefix, otherwise <see langword="false"/>.</returns>
+    public bool MediaTypeStartsWith(string prefix)
+        => MediaType?.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) is true;
+
     /// <summary>Sets <paramref name="mediaType"/> to null if it's empty or composed entirely of whitespace.</summary>
     private static void ValidateMediaType(ref string? mediaType)
     {
@@ -172,6 +180,7 @@ public string Uri
     /// as the instance actually contains all of the data it represents. If, however, the instance was constructed from another form of URI, one
     /// that simply references where the data can be found but doesn't actually contain the data, this property returns <see langword="false"/>.
     /// </remarks>
+    [MemberNotNullWhen(true, nameof(Data))]
     [JsonIgnore]
     public bool ContainsData => _dataUri is not null || _data is not null;
 
@@ -180,7 +189,6 @@ public string Uri
     /// If <see cref="ContainsData"/> is <see langword="true" />, this property returns the represented data.
     /// If <see cref="ContainsData"/> is <see langword="false" />, this property returns <see langword="null" />.
     /// </remarks>
-    [MemberNotNullWhen(true, nameof(ContainsData))]
     [JsonIgnore]
     public ReadOnlyMemory<byte>? Data
     {

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/DataContent.cs

@@ -484,12 +484,16 @@ private static List<ChatMessageContentItem> GetContentParts(IList<AIContent> con
                     parts.Add(new ChatMessageTextContentItem(textContent.Text));
                     break;
 
-                case ImageContent imageContent when imageContent.Data is { IsEmpty: false } data:
-                    parts.Add(new ChatMessageImageContentItem(BinaryData.FromBytes(data), imageContent.MediaType));
-                    break;
+                case DataContent dataContent when dataContent.MediaTypeStartsWith("image/"):
+                    if (dataContent.ContainsData)
+                    {
+                        parts.Add(new ChatMessageImageContentItem(BinaryData.FromBytes(dataContent.Data.Value), dataContent.MediaType));
+                    }
+                    else if (dataContent.Uri is string uri)
+                    {
+                        parts.Add(new ChatMessageImageContentItem(new Uri(uri)));
+                    }
 
-                case ImageContent imageContent when imageContent.Uri is string uri:
-                    parts.Add(new ChatMessageImageContentItem(new Uri(uri)));
                     break;
             }
         }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/ImageContent.cs

@@ -26,7 +26,7 @@
     <InjectSharedEmptyCollections>true</InjectSharedEmptyCollections>
     <InjectStringHashOnLegacy>true</InjectStringHashOnLegacy>
   </PropertyGroup>
-  
+
   <ItemGroup>
     <PackageReference Include="Azure.AI.Inference" />
     <PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
@@ -37,5 +37,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>

src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs

@@ -40,5 +40,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>

src/Libraries/Microsoft.Extensions.AI.AzureAIInference/Microsoft.Extensions.AI.AzureAIInference.csproj

@@ -386,74 +386,76 @@ private IEnumerable<OllamaChatRequestMessage> ToOllamaChatRequestMessages(ChatMe
         OllamaChatRequestMessage? currentTextMessage = null;
         foreach (var item in content.Contents)
         {
-            if (currentTextMessage is not null && item is not ImageContent)
+            if (item is DataContent { ContainsData: true } dataContent && dataContent.MediaTypeStartsWith("image/"))
             {
-                yield return currentTextMessage;
-                currentTextMessage = null;
-            }
-
-            switch (item)
-            {
-                case TextContent textContent:
-                    currentTextMessage = new OllamaChatRequestMessage
-                    {
-                        Role = content.Role.Value,
-                        Content = textContent.Text ?? string.Empty,
-                    };
-                    break;
-
-                case ImageContent imageContent when imageContent.Data is not null:
-                    IList<string> images = currentTextMessage?.Images ?? [];
-                    images.Add(Convert.ToBase64String(imageContent.Data.Value
+                IList<string> images = currentTextMessage?.Images ?? [];
+                images.Add(Convert.ToBase64String(dataContent.Data.Value
 #if NET
-                        .Span));
+                    .Span));
 #else
-                        .ToArray()));
+                    .ToArray()));
 #endif
 
-                    if (currentTextMessage is not null)
-                    {
-                        currentTextMessage.Images = images;
-                    }
-                    else
+                if (currentTextMessage is not null)
+                {
+                    currentTextMessage.Images = images;
+                }
+                else
+                {
+                    yield return new OllamaChatRequestMessage
                     {
-                        yield return new OllamaChatRequestMessage
+                        Role = content.Role.Value,
+                        Images = images,
+                    };
+                }
+            }
+            else
+            {
+                if (currentTextMessage is not null)
+                {
+                    yield return currentTextMessage;
+                    currentTextMessage = null;
+                }
+
+                switch (item)
+                {
+                    case TextContent textContent:
+                        currentTextMessage = new OllamaChatRequestMessage
                         {
                             Role = content.Role.Value,
-                            Images = images,
+                            Content = textContent.Text ?? string.Empty,
                         };
-                    }
+                        break;
 
-                    break;
-
-                case FunctionCallContent fcc:
-                {
-                    yield return new OllamaChatRequestMessage
+                    case FunctionCallContent fcc:
                     {
-                        Role = "assistant",
-                        Content = JsonSerializer.Serialize(new OllamaFunctionCallContent
+                        yield return new OllamaChatRequestMessage
                         {
-                            CallId = fcc.CallId,
-                            Name = fcc.Name,
-                            Arguments = JsonSerializer.SerializeToElement(fcc.Arguments, ToolCallJsonSerializerOptions.GetTypeInfo(typeof(IDictionary<string, object?>))),
-                        }, JsonContext.Default.OllamaFunctionCallContent)
-                    };
-                    break;
-                }
+                            Role = "assistant",
+                            Content = JsonSerializer.Serialize(new OllamaFunctionCallContent
+                            {
+                                CallId = fcc.CallId,
+                                Name = fcc.Name,
+                                Arguments = JsonSerializer.SerializeToElement(fcc.Arguments, ToolCallJsonSerializerOptions.GetTypeInfo(typeof(IDictionary<string, object?>))),
+                            }, JsonContext.Default.OllamaFunctionCallContent)
+                        };
+                        break;
+                    }
 
-                case FunctionResultContent frc:
-                {
-                    JsonElement jsonResult = JsonSerializer.SerializeToElement(frc.Result, ToolCallJsonSerializerOptions.GetTypeInfo(typeof(object)));
-                    yield return new OllamaChatRequestMessage
+                    case FunctionResultContent frc:
                     {
-                        Role = "tool",
-                        Content = JsonSerializer.Serialize(new OllamaFunctionResultContent
+                        JsonElement jsonResult = JsonSerializer.SerializeToElement(frc.Result, ToolCallJsonSerializerOptions.GetTypeInfo(typeof(object)));
+                        yield return new OllamaChatRequestMessage
                         {
-                            CallId = frc.CallId,
-                            Result = jsonResult,
-                        }, JsonContext.Default.OllamaFunctionResultContent)
-                    };
-                    break;
+                            Role = "tool",
+                            Content = JsonSerializer.Serialize(new OllamaFunctionResultContent
+                            {
+                                CallId = frc.CallId,
+                                Result = jsonResult,
+                            }, JsonContext.Default.OllamaFunctionResultContent)
+                        };
+                        break;
+                    }
                 }
             }
         }

src/Libraries/Microsoft.Extensions.AI.Ollama/Microsoft.Extensions.AI.Ollama.csproj

@@ -38,5 +38,5 @@
   <ItemGroup>
     <ProjectReference Include="../Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj" />
   </ItemGroup>
-  
+
 </Project>