Eliminate ingestion cache from AI Chat Web template (#6428)

* Begin updating to latest MEVD

* Reimplement JsonVectorStore to match updated MEVD APIs

* Remove ingestion cache and track ingestion status inside the vector DB

* Track the document metadata in a separate collection so we don't have to fetch literally everything from the vector DB in order to update ingestion

* Fix equality comparison issue with Qdrant connector

* Tidying

* More tidying

* Update MEAI.Templates test snapshots

---------

Co-authored-by: Jeff Handley <[email protected]>

Author: SteveSandersonMS

eng/Versions.props

@@ -165,9 +165,9 @@
     <CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion>9.4.1-beta.277</CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion>
     <CommunityToolkitAspireOllamaSharpVersion>9.4.1-beta.277</CommunityToolkitAspireOllamaSharpVersion>
     <MicrosoftExtensionsServiceDiscoveryVersion>9.2.0</MicrosoftExtensionsServiceDiscoveryVersion>
-    <MicrosoftSemanticKernelConnectorsAzureAISearchVersion>1.47.0-preview</MicrosoftSemanticKernelConnectorsAzureAISearchVersion>
-    <MicrosoftSemanticKernelConnectorsQdrantVersion>1.47.0-preview</MicrosoftSemanticKernelConnectorsQdrantVersion>
-    <MicrosoftSemanticKernelCoreVersion>1.47.0</MicrosoftSemanticKernelCoreVersion>
+    <MicrosoftSemanticKernelConnectorsAzureAISearchVersion>1.49.0-preview</MicrosoftSemanticKernelConnectorsAzureAISearchVersion>
+    <MicrosoftSemanticKernelConnectorsQdrantVersion>1.49.0-preview</MicrosoftSemanticKernelConnectorsQdrantVersion>
+    <MicrosoftSemanticKernelCoreVersion>1.49.0</MicrosoftSemanticKernelCoreVersion>
     <OllamaSharpVersion>5.1.13</OllamaSharpVersion>
     <OpenTelemetryVersion>1.9.0</OpenTelemetryVersion>
     <PdfPigVersion>0.1.9</PdfPigVersion>

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj.in

@@ -16,7 +16,6 @@
 <!--#if (UseQdrant)
     <PackageReference Include="Aspire.Hosting.Qdrant" Version="${AspireVersion}" />
 #endif -->
-    <PackageReference Include="CommunityToolkit.Aspire.Hosting.Sqlite" Version="${CommunityToolkitAspireHostingSqliteVersion}" />
 <!--#if (IsOllama)
     <PackageReference Include="CommunityToolkit.Aspire.Hosting.Ollama" Version="${CommunityToolkitAspireHostingOllamaVersion}" />
 #endif -->

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/Program.cs

@@ -38,8 +38,6 @@
 #else // UseLocalVectorStore
 #endif
 
-var ingestionCache = builder.AddSqlite("ingestionCache");
-
 var webApp = builder.AddProject<Projects.ChatWithCustomData_CSharp_Web>("aichatweb-app");
 #if (IsOllama) // AI SERVICE PROVIDER REFERENCES
 webApp
@@ -58,8 +56,5 @@
     .WaitFor(vectorDB);
 #else // UseLocalVectorStore
 #endif
-webApp
-    .WithReference(ingestionCache)
-    .WaitFor(ingestionCache);
 
 builder.Build().Run();

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj.in

@@ -30,11 +30,6 @@
 <!--#endif -->
 <!--#if (UseManagedIdentity) -->
     <PackageReference Include="Azure.Identity" Version="${AzureIdentityVersion}" />
-<!--#endif -->
-<!--#if (IsAspire) -->
-    <PackageReference Include="CommunityToolkit.Aspire.Microsoft.EntityFrameworkCore.Sqlite" Version="${CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion}" />
-<!--#else -->
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="${MicrosoftEntityFrameworkCoreSqliteVersion}" />
 <!--#endif -->
     <PackageReference Include="Microsoft.Extensions.AI" Version="${MicrosoftExtensionsAIVersion}" />
     <PackageReference Include="Microsoft.SemanticKernel.Core" Version="${MicrosoftSemanticKernelCoreVersion}" />

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Components/Pages/Chat/Chat.razor

@@ -118,7 +118,7 @@
         await InvokeAsync(StateHasChanged);
         var results = await Search.SearchAsync(searchPhrase, filenameFilter, maxResults: 5);
         return results.Select(result =>
-            $"<result filename=\"{result.FileName}\" page_number=\"{result.PageNumber}\">{result.Text}</result>");
+            $"<result filename=\"{result.DocumentId}\" page_number=\"{result.PageNumber}\">{result.Text}</result>");
     }
 
     public void Dispose()

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Program.Aspire.cs

@@ -53,10 +53,8 @@
 #endif
 builder.Services.AddScoped<DataIngestor>();
 builder.Services.AddSingleton<SemanticSearch>();
-builder.AddSqliteDbContext<IngestionCacheDbContext>("ingestionCache");
 
 var app = builder.Build();
-IngestionCacheDbContext.Initialize(app.Services);
 
 app.MapDefaultEndpoints();
 

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Program.cs

@@ -1,4 +1,3 @@
-using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.VectorData;
 using ChatWithCustomData_CSharp.Web.Components;
@@ -102,11 +101,7 @@
 builder.Services.AddChatClient(chatClient).UseFunctionInvocation().UseLogging();
 builder.Services.AddEmbeddingGenerator(embeddingGenerator);
 
-builder.Services.AddDbContext<IngestionCacheDbContext>(options =>
-    options.UseSqlite("Data Source=ingestioncache.db"));
-
 var app = builder.Build();
-IngestionCacheDbContext.Initialize(app.Services);
 
 // Configure the HTTP request pipeline.
 if (!app.Environment.IsDevelopment())

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/SemanticSearchRecord.cs renamed to src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/IngestedChunk.cs

@@ -2,7 +2,7 @@
 
 namespace ChatWithCustomData_CSharp.Web.Services;
 
-public class SemanticSearchRecord
+public class IngestedChunk
 {
     [VectorStoreRecordKey]
 #if (UseQdrant)
@@ -11,8 +11,8 @@ public class SemanticSearchRecord
     public required string Key { get; set; }
 #endif
 
-    [VectorStoreRecordData(IsFilterable = true)]
-    public required string FileName { get; set; }
+    [VectorStoreRecordData(IsIndexed = true)]
+    public required string DocumentId { get; set; }
 
     [VectorStoreRecordData]
     public int PageNumber { get; set; }
@@ -21,9 +21,9 @@ public class SemanticSearchRecord
     public required string Text { get; set; }
 
 #if (IsOllama)
-    [VectorStoreRecordVector(384, DistanceFunction.CosineSimilarity)] // 384 is the default vector size for the all-minilm embedding model
+    [VectorStoreRecordVector(384, DistanceFunction = DistanceFunction.CosineSimilarity)] // 384 is the default vector size for the all-minilm embedding model
 #else
-    [VectorStoreRecordVector(1536, DistanceFunction.CosineSimilarity)] // 1536 is the default vector size for the OpenAI text-embedding-3-small model
+    [VectorStoreRecordVector(1536, DistanceFunction = DistanceFunction.CosineSimilarity)] // 1536 is the default vector size for the OpenAI text-embedding-3-small model
 #endif
     public ReadOnlyMemory<float> Vector { get; set; }
 }

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/IngestedDocument.cs

@@ -0,0 +1,26 @@
+using Microsoft.Extensions.VectorData;
+
+namespace ChatWithCustomData_CSharp.Web.Services;
+
+public class IngestedDocument
+{
+    [VectorStoreRecordKey]
+#if (UseQdrant)
+    public required Guid Key { get; set; }
+#else
+    public required string Key { get; set; }
+#endif
+
+    [VectorStoreRecordData(IsIndexed = true)]
+    public required string SourceId { get; set; }
+
+    [VectorStoreRecordData]
+    public required string DocumentId { get; set; }
+
+    [VectorStoreRecordData]
+    public required string DocumentVersion { get; set; }
+
+    // The vector is not used but required for some vector databases
+    [VectorStoreRecordVector(1, DistanceFunction = DistanceFunction.CosineSimilarity)]
+    public ReadOnlyMemory<float> Vector { get; set; } = new ReadOnlyMemory<float>([0]);
+}

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/Ingestion/DataIngestor.cs

@@ -1,14 +1,12 @@
-using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.AI;
+using Microsoft.Extensions.AI;
 using Microsoft.Extensions.VectorData;
 
 namespace ChatWithCustomData_CSharp.Web.Services.Ingestion;
 
 public class DataIngestor(
     ILogger<DataIngestor> logger,
     IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
-    IVectorStore vectorStore,
-    IngestionCacheDbContext ingestionCacheDb)
+    IVectorStore vectorStore)
 {
     public static async Task IngestDataAsync(IServiceProvider services, IIngestionSource source)
     {
@@ -20,48 +18,48 @@ public static async Task IngestDataAsync(IServiceProvider services, IIngestionSo
     public async Task IngestDataAsync(IIngestionSource source)
     {
 #if (UseQdrant)
-        var vectorCollection = vectorStore.GetCollection<Guid, SemanticSearchRecord>("data-ChatWithCustomData-CSharp.Web-ingestion");
+        var chunksCollection = vectorStore.GetCollection<Guid, IngestedChunk>("data-ChatWithCustomData-CSharp.Web-chunks");
+        var documentsCollection = vectorStore.GetCollection<Guid, IngestedDocument>("data-ChatWithCustomData-CSharp.Web-documents");
 #else
-        var vectorCollection = vectorStore.GetCollection<string, SemanticSearchRecord>("data-ChatWithCustomData-CSharp.Web-ingestion");
+        var chunksCollection = vectorStore.GetCollection<string, IngestedChunk>("data-ChatWithCustomData-CSharp.Web-chunks");
+        var documentsCollection = vectorStore.GetCollection<string, IngestedDocument>("data-ChatWithCustomData-CSharp.Web-documents");
 #endif
-        await vectorCollection.CreateCollectionIfNotExistsAsync();
+        await chunksCollection.CreateCollectionIfNotExistsAsync();
+        await documentsCollection.CreateCollectionIfNotExistsAsync();
 
-        var documentsForSource = ingestionCacheDb.Documents
-            .Where(d => d.SourceId == source.SourceId)
-            .Include(d => d.Records);
+        var sourceId = source.SourceId;
+        var documentsForSource = await documentsCollection.GetAsync(doc => doc.SourceId == sourceId, top: int.MaxValue).ToListAsync();
 
-        var deletedFiles = await source.GetDeletedDocumentsAsync(documentsForSource);
-        foreach (var deletedFile in deletedFiles)
+        var deletedDocuments = await source.GetDeletedDocumentsAsync(documentsForSource);
+        foreach (var deletedDocument in deletedDocuments)
         {
-            logger.LogInformation("Removing ingested data for {file}", deletedFile.Id);
-            await vectorCollection.DeleteBatchAsync(deletedFile.Records.Select(r => r.Id));
-            ingestionCacheDb.Documents.Remove(deletedFile);
+            logger.LogInformation("Removing ingested data for {documentId}", deletedDocument.DocumentId);
+            await DeleteChunksForDocumentAsync(deletedDocument);
+            await documentsCollection.DeleteAsync(deletedDocument.Key);
         }
-        await ingestionCacheDb.SaveChangesAsync();
 
-        var modifiedDocs = await source.GetNewOrModifiedDocumentsAsync(documentsForSource);
-        foreach (var modifiedDoc in modifiedDocs)
+        var modifiedDocuments = await source.GetNewOrModifiedDocumentsAsync(documentsForSource);
+        foreach (var modifiedDocument in modifiedDocuments)
         {
-            logger.LogInformation("Processing {file}", modifiedDoc.Id);
+            logger.LogInformation("Processing {documentId}", modifiedDocument.DocumentId);
+            await DeleteChunksForDocumentAsync(modifiedDocument);
 
-            if (modifiedDoc.Records.Count > 0)
-            {
-                await vectorCollection.DeleteBatchAsync(modifiedDoc.Records.Select(r => r.Id));
-            }
+            await documentsCollection.UpsertAsync(modifiedDocument);
 
-            var newRecords = await source.CreateRecordsForDocumentAsync(embeddingGenerator, modifiedDoc.Id);
-            await foreach (var id in vectorCollection.UpsertBatchAsync(newRecords)) { }
+            var newRecords = await source.CreateChunksForDocumentAsync(embeddingGenerator, modifiedDocument);
+            await chunksCollection.UpsertAsync(newRecords);
+        }
 
-            modifiedDoc.Records.Clear();
-            modifiedDoc.Records.AddRange(newRecords.Select(r => new IngestedRecord { Id = r.Key, DocumentId = modifiedDoc.Id }));
+        logger.LogInformation("Ingestion is up-to-date");
 
-            if (ingestionCacheDb.Entry(modifiedDoc).State == EntityState.Detached)
+        async Task DeleteChunksForDocumentAsync(IngestedDocument document)
+        {
+            var documentId = document.DocumentId;
+            var chunksToDelete = await chunksCollection.GetAsync(record => record.DocumentId == documentId, int.MaxValue).ToListAsync();
+            if (chunksToDelete.Any())
             {
-                ingestionCacheDb.Documents.Add(modifiedDoc);
+                await chunksCollection.DeleteAsync(chunksToDelete.Select(r => r.Key));
             }
         }
-
-        await ingestionCacheDb.SaveChangesAsync();
-        logger.LogInformation("Ingestion is up-to-date");
     }
 }