Skip to content

Commit 9433c9b

Browse files
floberndflobernd
authored
Update connector (#12)
1 parent b75c240 commit 9433c9b

34 files changed

+1541
-2075
lines changed

‎Directory.Packages.props

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,18 @@
55
<CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
66
</PropertyGroup>
77
<ItemGroup>
8-
<PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.16.2" />
9-
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.30.0" />
8+
<PackageVersion Include="Elastic.Clients.Elasticsearch" Version="8.18.0" />
9+
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="9.5.0-preview.1.25262.9" />
10+
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="9.0.5" />
11+
<PackageVersion Include="Microsoft.SemanticKernel.Abstractions" Version="1.51.0" />
1012
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
1113
<PackageVersion Include="MinVer" Version="6.0.0" />
12-
<PackageVersion Include="System.Text.Json" Version="8.0.5" />
14+
<PackageVersion Include="PolySharp" Version="1.15.0" />
15+
<PackageVersion Include="System.Text.Json" Version="9.0.5" />
1316
<!-- Playground -->
14-
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.1" />
15-
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.30.0" />
16-
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.30.0" />
17+
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="9.0.5" />
18+
<PackageVersion Include="Microsoft.SemanticKernel.Connectors.AzureOpenAI" Version="1.51.0" />
19+
<PackageVersion Include="Microsoft.SemanticKernel.PromptTemplates.Handlebars" Version="1.51.0" />
1720
</ItemGroup>
1821
<ItemGroup>
1922
<GlobalPackageReference Include="Microsoft.Build.CopyOnWrite" Version="1.0.334" />

‎Elastic.SemanticKernel.Connectors.Elasticsearch/Elastic.SemanticKernel.Connectors.Elasticsearch.csproj

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,14 @@
3434

3535
<ItemGroup>
3636
<PackageReference Include="Elastic.Clients.Elasticsearch" />
37+
<PackageReference Include="Microsoft.Extensions.AI.Abstractions" />
38+
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
3739
<PackageReference Include="Microsoft.SemanticKernel.Abstractions" />
38-
<PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="all" />
3940
<PackageReference Include="MinVer" />
41+
<PackageReference Include="PolySharp">
42+
<PrivateAssets>all</PrivateAssets>
43+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
44+
</PackageReference>
4045
<PackageReference Include="System.Text.Json" />
4146
</ItemGroup>
4247
</Project>

‎Elastic.SemanticKernel.Connectors.Elasticsearch/ElasticsearchConstants.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
namespace Elastic.SemanticKernel.Connectors.Elasticsearch;
2+
3+
internal static class ElasticsearchConstants
4+
{
5+
internal const string VectorStoreSystemName = "elasticsearch";
6+
}

‎Elastic.SemanticKernel.Connectors.Elasticsearch/ElasticsearchDataModelMapper.cs

Lines changed: 50 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,67 +2,87 @@
22
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information.
44

5-
using System.Collections.Generic;
6-
using System.Linq;
5+
using System.Text.Json;
76
using System.Text.Json.Nodes;
87

98
using Elastic.Clients.Elasticsearch;
109
using Elastic.Transport.Extensions;
1110

11+
using Microsoft.Extensions.AI;
1212
using Microsoft.Extensions.VectorData;
13+
using Microsoft.Extensions.VectorData.ConnectorSupport;
1314
using Microsoft.SemanticKernel;
1415

1516
namespace Elastic.SemanticKernel.Connectors.Elasticsearch;
1617

1718
/// <summary>
18-
/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under,
19-
/// within Elasticsearch.
19+
/// A mapper that maps between the generic Semantic Kernel data model and the model that the data is stored under,
20+
/// within Elasticsearch.
2021
/// </summary>
21-
internal sealed class ElasticsearchDataModelMapper<TRecord> :
22-
IVectorStoreRecordMapper<TRecord, (string? id, JsonObject document)>
22+
internal sealed class ElasticsearchDataModelMapper<TKey, TRecord> :
23+
IElasticsearchVectorStoreRecordMapper<TRecord, (string? id, JsonObject document)>
24+
where TKey : notnull
25+
where TRecord : notnull
2326
{
24-
/// <summary>The Elasticsearch client settings.</summary>
25-
private readonly IElasticsearchClientSettings _elasticsearchClientSettings;
27+
/// <summary>
28+
/// A model representing a record in a vector store collection.
29+
/// </summary>
30+
private readonly VectorStoreRecordModel _model;
2631

27-
/// <summary>A mapping from <see cref="VectorStoreRecordDefinition" /> to storage model property name.</summary>
28-
private readonly Dictionary<VectorStoreRecordProperty, string> _propertyToStorageName;
32+
/// <summary>
33+
/// The Elasticsearch client settings.
34+
/// </summary>
35+
private readonly IElasticsearchClientSettings _elasticsearchClientSettings;
2936

3037
/// <summary>
31-
/// Initializes a new instance of the <see cref="ElasticsearchGenericDataModelMapper" /> class.
38+
/// Initializes a new instance of the <see cref="ElasticsearchGenericDataModelMapper" /> class.
3239
/// </summary>
33-
/// <param name="propertyToStorageName">A mapping from <see cref="VectorStoreRecordDefinition" /> to storage model property name.</param>
40+
/// <param name="model">A model representing a record in a vector store collection.</param>
3441
/// <param name="elasticsearchClientSettings">The Elasticsearch client settings to use.</param>
3542
public ElasticsearchDataModelMapper(
36-
Dictionary<VectorStoreRecordProperty, string> propertyToStorageName,
43+
VectorStoreRecordModel model,
3744
IElasticsearchClientSettings elasticsearchClientSettings)
3845
{
39-
Verify.NotNull(propertyToStorageName);
46+
Verify.NotNull(model);
4047
Verify.NotNull(elasticsearchClientSettings);
4148

4249
// Assign.
50+
_model = model;
4351
_elasticsearchClientSettings = elasticsearchClientSettings;
44-
_propertyToStorageName = propertyToStorageName;
4552
}
4653

4754
/// <inheritdoc />
48-
public (string? id, JsonObject document) MapFromDataToStorageModel(TRecord dataModel)
55+
public (string? id, JsonObject document) MapFromDataToStorageModel(TRecord dataModel, Embedding<float>?[]? generatedEmbeddings)
4956
{
50-
// Serialize the whole record to JsonObject.
57+
Verify.NotNull(dataModel);
5158

52-
var document = SerializeSource(dataModel, _elasticsearchClientSettings)!;
59+
// Serialize the whole record to JsonObject.
5360

54-
// Extract key property.
61+
var document = SerializeSource(dataModel, _elasticsearchClientSettings);
5562

56-
var keyProperty = _propertyToStorageName.Single(x => x.Key is VectorStoreRecordKeyProperty);
57-
var keyValue = document[keyProperty.Value]!.AsValue();
63+
// Extract key property and remove it from document.
5864

59-
var id = keyValue.GetValue<string?>();
65+
var keyProperty = _model.KeyProperty;
66+
var id = (TKey?)keyProperty.GetValueAsObject(dataModel);
67+
document.Remove(keyProperty.StorageName);
6068

61-
// Remove key property from document.
69+
// Update vector properties with generated embeddings.
6270

63-
document.Remove(keyProperty.Value);
71+
if (generatedEmbeddings is not null)
72+
{
73+
for (var i = 0; i < _model.VectorProperties.Count; ++i)
74+
{
75+
if (generatedEmbeddings[i] is not { } embedding)
76+
{
77+
continue;
78+
}
79+
80+
var vectorProperty = _model.VectorProperties[i];
81+
document[vectorProperty.StorageName] = JsonValue.Create(embedding.Vector);
82+
}
83+
}
6484

65-
return (id, document);
85+
return ((id is null) ? null : ElasticsearchVectorStoreRecordFieldMapping.KeyToElasticsearchId(id), document);
6686
}
6787

6888
/// <inheritdoc />
@@ -71,19 +91,19 @@ public TRecord MapFromStorageToDataModel((string? id, JsonObject document) stora
7191
{
7292
// Add key property to document.
7393

74-
var keyProperty = _propertyToStorageName.Single(x => x.Key is VectorStoreRecordKeyProperty);
75-
storageModel.document.Add(keyProperty.Value, storageModel.id);
94+
var keyProperty = _model.KeyProperty;
95+
storageModel.document.Add(keyProperty.StorageName, JsonValue.Create(ElasticsearchVectorStoreRecordFieldMapping.ElasticsearchIdToKey<TKey>(storageModel.id!)));
7696

7797
// Serialize the whole model into the user-defined record type.
7898

7999
return _elasticsearchClientSettings.SourceSerializer.Deserialize<TRecord>(storageModel.document)!;
80100
}
81101

82-
private static JsonObject? SerializeSource<T>(T? obj, IElasticsearchClientSettings settings)
102+
private static JsonObject SerializeSource<T>(T obj, IElasticsearchClientSettings settings)
83103
{
84-
if (obj is null)
104+
if (settings.SourceSerializer.TryGetJsonSerializerOptions(out var options))
85105
{
86-
return null;
106+
return (JsonObject)JsonSerializer.SerializeToNode(obj, options)!;
87107
}
88108

89109
using var stream = settings.MemoryStreamFactory.Create();

0 commit comments

Comments
 (0)