Update connector (#12)

Author: flobernd

Directory.Packages.props

@@ -5,15 +5,18 @@
     <CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
   </PropertyGroup>
   <ItemGroup>
-    <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.16.2" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.30.0" />
+    <PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.18.0" />
+    <PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.5.0-preview.1.25262.9" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.5" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.51.0" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <PackageVersion Include="MinVer" Version="6.0.0" />
-    <PackageVersion Include="System.Text.Json" Version="8.0.5" />
+    <PackageVersion Include="PolySharp" Version="1.15.0" />
+    <PackageVersion Include="System.Text.Json" Version="9.0.5" />
     <!-- Playground -->
-    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
-    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.30.0" />
-    <PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.30.0" />
+    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.5" />
+    <PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.51.0" />
+    <PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.51.0" />
   </ItemGroup>
   <ItemGroup>
     <GlobalPackageReference Include="Microsoft.Build.CopyOnWrite" Version="1.0.334" />

Elastic.SemanticKernel.Connectors.Elasticsearch/Elastic.SemanticKernel.Connectors.Elasticsearch.csproj

@@ -34,9 +34,14 @@
 
   <ItemGroup>
     <PackageReference Include="Elastic.Clients.Elasticsearch" />
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
     <PackageReference Include="Microsoft.SemanticKernel.Abstractions" />
-    <PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="all" />
     <PackageReference Include="MinVer" />
+    <PackageReference Include="PolySharp">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="System.Text.Json" />
   </ItemGroup>
 </Project>

Elastic.SemanticKernel.Connectors.Elasticsearch/ElasticsearchConstants.cs

@@ -0,0 +1,6 @@
+namespace Elastic.SemanticKernel.Connectors.Elasticsearch;
+
+internal static class ElasticsearchConstants
+{
+    internal const string VectorStoreSystemName = "elasticsearch";
+}

Elastic.SemanticKernel.Connectors.Elasticsearch/ElasticsearchDataModelMapper.cs

@@ -2,67 +2,87 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-using System.Collections.Generic;
-using System.Linq;
+using System.Text.Json;
 using System.Text.Json.Nodes;
 
 using Elastic.Clients.Elasticsearch;
 using Elastic.Transport.Extensions;
 
+using Microsoft.Extensions.AI;
 using Microsoft.Extensions.VectorData;
+using Microsoft.Extensions.VectorData.ConnectorSupport;
 using Microsoft.SemanticKernel;
 
 namespace Elastic.SemanticKernel.Connectors.Elasticsearch;
 
 /// <summary>
-///     A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under,
-///     within Elasticsearch.
+/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under,
+/// within Elasticsearch.
 /// </summary>
-internal sealed class ElasticsearchDataModelMapper<TRecord> :
-    IVectorStoreRecordMapper<TRecord, (string? id, JsonObject document)>
+internal sealed class ElasticsearchDataModelMapper<TKey, TRecord> :
+    IElasticsearchVectorStoreRecordMapper<TRecord, (string? id, JsonObject document)>
+    where TKey : notnull
+    where TRecord : notnull
 {
-    /// <summary>The Elasticsearch client settings.</summary>
-    private readonly IElasticsearchClientSettings _elasticsearchClientSettings;
+    /// <summary>
+    /// A model representing a record in a vector store collection.
+    /// </summary>
+    private readonly VectorStoreRecordModel _model;
 
-    /// <summary>A mapping from <see cref="VectorStoreRecordDefinition" /> to storage model property name.</summary>
-    private readonly Dictionary<VectorStoreRecordProperty, string> _propertyToStorageName;
+    /// <summary>
+    /// The Elasticsearch client settings.
+    /// </summary>
+    private readonly IElasticsearchClientSettings _elasticsearchClientSettings;
 
     /// <summary>
-    ///     Initializes a new instance of the <see cref="ElasticsearchGenericDataModelMapper" /> class.
+    /// Initializes a new instance of the <see cref="ElasticsearchGenericDataModelMapper" /> class.
     /// </summary>
-    /// <param name="propertyToStorageName">A mapping from <see cref="VectorStoreRecordDefinition" /> to storage model property name.</param>
+    /// <param name="model">A model representing a record in a vector store collection.</param>
     /// <param name="elasticsearchClientSettings">The Elasticsearch client settings to use.</param>
     public ElasticsearchDataModelMapper(
-        Dictionary<VectorStoreRecordProperty, string> propertyToStorageName,
+        VectorStoreRecordModel model,
         IElasticsearchClientSettings elasticsearchClientSettings)
     {
-        Verify.NotNull(propertyToStorageName);
+        Verify.NotNull(model);
         Verify.NotNull(elasticsearchClientSettings);
 
         // Assign.
+        _model = model;
         _elasticsearchClientSettings = elasticsearchClientSettings;
-        _propertyToStorageName = propertyToStorageName;
     }
 
     /// <inheritdoc />
-    public (string? id, JsonObject document) MapFromDataToStorageModel(TRecord dataModel)
+    public (string? id, JsonObject document) MapFromDataToStorageModel(TRecord dataModel, Embedding<float>?[]? generatedEmbeddings)
     {
-        // Serialize the whole record to JsonObject.
+        Verify.NotNull(dataModel);
 
-        var document = SerializeSource(dataModel, _elasticsearchClientSettings)!;
+        // Serialize the whole record to JsonObject.
 
-        // Extract key property.
+        var document = SerializeSource(dataModel, _elasticsearchClientSettings);
 
-        var keyProperty = _propertyToStorageName.Single(x => x.Key is VectorStoreRecordKeyProperty);
-        var keyValue = document[keyProperty.Value]!.AsValue();
+        // Extract key property and remove it from document.
 
-        var id = keyValue.GetValue<string?>();
+        var keyProperty = _model.KeyProperty;
+        var id = (TKey?)keyProperty.GetValueAsObject(dataModel);
+        document.Remove(keyProperty.StorageName);
 
-        // Remove key property from document.
+        // Update vector properties with generated embeddings.
 
-        document.Remove(keyProperty.Value);
+        if (generatedEmbeddings is not null)
+        {
+            for (var i = 0; i < _model.VectorProperties.Count; ++i)
+            {
+                if (generatedEmbeddings[i] is not { } embedding)
+                {
+                    continue;
+                }
+
+                var vectorProperty = _model.VectorProperties[i];
+                document[vectorProperty.StorageName] = JsonValue.Create(embedding.Vector);
+            }
+        }
 
-        return (id, document);
+        return ((id is null) ? null : ElasticsearchVectorStoreRecordFieldMapping.KeyToElasticsearchId(id), document);
     }
 
     /// <inheritdoc />
@@ -71,19 +91,19 @@ public TRecord MapFromStorageToDataModel((string? id, JsonObject document) stora
     {
         // Add key property to document.
 
-        var keyProperty = _propertyToStorageName.Single(x => x.Key is VectorStoreRecordKeyProperty);
-        storageModel.document.Add(keyProperty.Value, storageModel.id);
+        var keyProperty = _model.KeyProperty;
+        storageModel.document.Add(keyProperty.StorageName, JsonValue.Create(ElasticsearchVectorStoreRecordFieldMapping.ElasticsearchIdToKey<TKey>(storageModel.id!)));
 
         // Serialize the whole model into the user-defined record type.
 
         return _elasticsearchClientSettings.SourceSerializer.Deserialize<TRecord>(storageModel.document)!;
     }
 
-    private static JsonObject? SerializeSource<T>(T? obj, IElasticsearchClientSettings settings)
+    private static JsonObject SerializeSource<T>(T obj, IElasticsearchClientSettings settings)
     {
-        if (obj is null)
+        if (settings.SourceSerializer.TryGetJsonSerializerOptions(out var options))
         {
-            return null;
+            return (JsonObject)JsonSerializer.SerializeToNode(obj, options)!;
         }
 
         using var stream = settings.MemoryStreamFactory.Create();