Update to M.E.AI 9.3.0-preview.1.25161.3 (#1317)

Author: stephentoub

nuget/Microsoft.ML.OnnxRuntimeGenAI.Managed.nuspec

@@ -14,19 +14,19 @@
     <tags>ONNX;ONNX Runtime;ONNX Runtime Gen AI;Machine Learning</tags>
     <dependencies>
       <group targetFramework="net8.0">
-        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25114.11" />
+        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25161.3" />
       </group>
       <group targetFramework="netstandard2.0">
-        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25114.11" />
+        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25161.3" />
       </group>
       <group targetFramework="net8.0-android31.0">
-        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25114.11" />
+        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25161.3" />
       </group>
       <group targetFramework="net8.0-ios15.4">
-        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25114.11" />
+        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25161.3" />
       </group>
       <group targetFramework="net8.0-maccatalyst14.0">
-        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25114.11" />
+        <dependency id="Microsoft.Extensions.AI.Abstractions" version="9.3.0-preview.1.25161.3" />
       </group>
     </dependencies>
   </metadata>

src/csharp/Microsoft.ML.OnnxRuntimeGenAI.csproj

@@ -122,7 +122,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25114.11" />
+    <PackageReference Include="Microsoft.Extensions.AI.Abstractions" Version="9.3.0-preview.1.25161.3" />
   </ItemGroup>
 
 </Project>

src/csharp/OnnxRuntimeGenAIChatClient.cs

@@ -2,9 +2,9 @@
 // Licensed under the MIT License.
 
 using System;
-using System.Buffers;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Extensions.AI;
@@ -16,8 +16,8 @@ namespace Microsoft.ML.OnnxRuntimeGenAI;
 /// <summary>Provides an <see cref="IChatClient"/> implementation for interacting with an ONNX Runtime GenAI <see cref="Model"/>.</summary>
 public sealed class OnnxRuntimeGenAIChatClient : IChatClient
 {
-    /// <summary>The options used to configure the instance.</summary>
-    private readonly OnnxRuntimeGenAIChatClientOptions _options;
+    /// <summary>Options used to configure the instance's behavior.</summary>
+    private readonly OnnxRuntimeGenAIChatClientOptions? _options;
     /// <summary>The wrapped <see cref="Model"/>.</summary>
     private readonly Model _model;
     /// <summary>The wrapped <see cref="Tokenizer"/>.</summary>
@@ -32,58 +32,44 @@ public sealed class OnnxRuntimeGenAIChatClient : IChatClient
     private CachedGenerator? _cachedGenerator;
 
     /// <summary>Initializes an instance of the <see cref="OnnxRuntimeGenAIChatClient"/> class.</summary>
-    /// <param name="options">Options used to configure the client instance.</param>
     /// <param name="modelPath">The file path to the model to load.</param>
-    /// <exception cref="ArgumentNullException"><paramref name="options"/> is <see langword="null"/>.</exception>
+    /// <param name="options">Options used to configure the client instance.</param>
     /// <exception cref="ArgumentNullException"><paramref name="modelPath"/> is <see langword="null"/>.</exception>
-    public OnnxRuntimeGenAIChatClient(OnnxRuntimeGenAIChatClientOptions options, string modelPath)
+    public OnnxRuntimeGenAIChatClient(string modelPath, OnnxRuntimeGenAIChatClientOptions? options = null)
     {
-        if (options is null)
-        {
-            throw new ArgumentNullException(nameof(options));
-        }
-
         if (modelPath is null)
         {
             throw new ArgumentNullException(nameof(modelPath));
         }
 
-        _options = options;
-
         _ownsModel = true;
         _model = new Model(modelPath);
         _tokenizer = new Tokenizer(_model);
+        _options = options;
 
         _metadata = new("onnx", new Uri($"file://{modelPath}"), modelPath);
     }
 
     /// <summary>Initializes an instance of the <see cref="OnnxRuntimeGenAIChatClient"/> class.</summary>
-    /// <param name="options">Options used to configure the client instance.</param>
     /// <param name="model">The model to employ.</param>
     /// <param name="ownsModel">
     /// <see langword="true"/> if this <see cref="IChatClient"/> owns the <paramref name="model"/> and should
     /// dispose of it when this <see cref="IChatClient"/> is disposed; otherwise, <see langword="false"/>.
     /// The default is <see langword="true"/>.
     /// </param>
-    /// <exception cref="ArgumentNullException"><paramref name="options"/> is <see langword="null"/>.</exception>
+    /// <param name="options">Options used to configure the client instance.</param>
     /// <exception cref="ArgumentNullException"><paramref name="model"/> is <see langword="null"/>.</exception>
-    public OnnxRuntimeGenAIChatClient(OnnxRuntimeGenAIChatClientOptions options, Model model, bool ownsModel = true)
+    public OnnxRuntimeGenAIChatClient(Model model, bool ownsModel = true, OnnxRuntimeGenAIChatClientOptions? options = null)
     {
-        if (options is null)
-        {
-            throw new ArgumentNullException(nameof(options));
-        }
-
         if (model is null)
         {
             throw new ArgumentNullException(nameof(model));
         }
 
-        _options = options;
-
         _ownsModel = ownsModel;
         _model = model;
         _tokenizer = new Tokenizer(_model);
+        _options = options;
 
         _metadata = new("onnx");
     }
@@ -106,16 +92,16 @@ public void Dispose()
 
     /// <inheritdoc/>
     public Task<ChatResponse> GetResponseAsync(
-        IList<ChatMessage> chatMessages, ChatOptions? options = null, CancellationToken cancellationToken = default) =>
-        GetStreamingResponseAsync(chatMessages, options, cancellationToken).ToChatResponseAsync(cancellationToken: cancellationToken);
+        IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default) =>
+        GetStreamingResponseAsync(messages, options, cancellationToken).ToChatResponseAsync(cancellationToken: cancellationToken);
 
     /// <inheritdoc/>
     public async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
-        IList<ChatMessage> chatMessages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+        IEnumerable<ChatMessage> messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
     {
-        if (chatMessages is null)
+        if (messages is null)
         {
-            throw new ArgumentNullException(nameof(chatMessages));
+            throw new ArgumentNullException(nameof(messages));
         }
 
         // Check to see whether there's a cached generator. If there is, and if its id matches what we got from the client,
@@ -133,12 +119,18 @@ generator.ChatThreadId is null ||
         }
 
         // If caching is enabled, generate a new ID to represent the state of the generator when we finish this response.
-        generator.ChatThreadId = _options.EnableCaching ? Guid.NewGuid().ToString("N") : null;
+        generator.ChatThreadId = _options?.EnableCaching is true ? Guid.NewGuid().ToString("N") : null;
 
         // Format and tokenize the message.
-        using Sequences tokens = _tokenizer.Encode(_options.PromptFormatter(chatMessages, options));
+        string formattedPrompt = _options?.PromptFormatter is { } formatter ?
+            formatter(messages, options) :
+            FormatPromptDefault(messages, options);
+
+        using Sequences tokens = _tokenizer.Encode(formattedPrompt);
         try
         {
+            string responseId = Guid.NewGuid().ToString("N");
+
             generator.Generator.AppendTokenSequences(tokens);
             int inputTokens = tokens[0].Length, outputTokens = 0;
 
@@ -172,11 +164,10 @@ generator.ChatThreadId is null ||
 
                 // Yield the next token in the stream.
                 outputTokens++;
-                yield return new()
+                yield return new(ChatRole.Assistant, next)
                 {
                     CreatedAt = DateTimeOffset.UtcNow,
-                    Role = ChatRole.Assistant,
-                    Text = next,
+                    ResponseId = responseId,
                 };
             }
 
@@ -193,7 +184,7 @@ generator.ChatThreadId is null ||
                 CreatedAt = DateTimeOffset.UtcNow,
                 FinishReason = options is not null && options.MaxOutputTokens <= outputTokens ? ChatFinishReason.Length : ChatFinishReason.Stop,
                 ModelId = _metadata.ModelId,
-                ResponseId = Guid.NewGuid().ToString(),
+                ResponseId = responseId,
                 Role = ChatRole.Assistant,
             };
         }
@@ -228,7 +219,19 @@ generator.ChatThreadId is null ||
     /// <summary>Gets whether the specified token is a stop sequence.</summary>
     private bool IsStop(string token, ChatOptions? options) =>
         options?.StopSequences?.Contains(token) is true ||
-        _options.StopSequences.Contains(token);
+        _options?.StopSequences?.Contains(token) is true;
+
+    /// <summary>Formats messages into a prompt using a default format.</summary>
+    private static string FormatPromptDefault(IEnumerable<ChatMessage> messages, ChatOptions? options)
+    {
+        StringBuilder sb = new();
+        foreach (var message in messages)
+        {
+            sb.Append(message).AppendLine();
+}
+
+        return sb.ToString();
+    }
 
     /// <summary>Updates the <paramref name="generatorParams"/> based on the supplied <paramref name="options"/>.</summary>
     private static void UpdateGeneratorParamsFromOptions(GeneratorParams generatorParams, ChatOptions? options)

src/csharp/OnnxRuntimeGenAIChatClientOptions.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Text;
 using Microsoft.Extensions.AI;
 
 #nullable enable
@@ -36,17 +35,6 @@ public sealed class OnnxRuntimeGenAIChatClientOptions
 {
     private IList<string> _stopSequences = [];
 
-    private Func<IEnumerable<ChatMessage>, ChatOptions?, string> _promptFormatter = static (messages, _) =>
-    {
-        StringBuilder sb = new();
-        foreach (var message in messages)
-        {
-            sb.Append(message).AppendLine();
-        }
-
-        return sb.ToString();
-    };
-
     /// <summary>Initializes a new instance of the <see cref="OnnxRuntimeGenAIChatClientOptions"/> class.</summary>
     /// <param name="stopSequences">The stop sequences used by the model.</param>
     /// <param name="promptFormatter">The function to use to format a list of messages for input into the model.</param>
@@ -62,25 +50,18 @@ public OnnxRuntimeGenAIChatClientOptions()
     /// <remarks>
     /// These will apply in addition to any stop sequences that are a part of the <see cref="ChatOptions.StopSequences"/>
     /// provided to the <see cref="IChatClient.GetResponseAsync"/> and <see cref="IChatClient.GetStreamingResponseAsync"/>
-    /// methods.
+    /// methods. If <see langword="null"/>, this will not contribute any additional stop sequences.
     /// </remarks>
-    public IList<string> StopSequences
-    {
-        get => _stopSequences;
-        set => _stopSequences = value ?? throw new ArgumentNullException(nameof(value));
-    }
+    public IList<string>? StopSequences { get; set; }
 
     /// <summary>Gets or sets a delegate that formats a prompt string from a list of chat messages.</summary>
     /// <remarks>
     /// Each time <see cref="IChatClient.GetResponseAsync"/> or <see cref="IChatClient.GetStreamingResponseAsync"/>
     /// is invoked, this delegate will be invoked with the supplied list of messages to produce a string that
-    /// will be tokenized and provided to the underlying <see cref="Generator"/>.
+    /// will be tokenized and provided to the underlying <see cref="Generator"/>. If <see langword="null"/>,
+    /// the <see cref="OnnxRuntimeGenAIChatClient"/> will choose a default prompt formatter to employ.
     /// </remarks>
-    public Func<IEnumerable<ChatMessage>, ChatOptions?, string> PromptFormatter
-    {
-        get => _promptFormatter;
-        set => _promptFormatter = value ?? throw new ArgumentNullException(nameof(value));
-    }
+    public Func<IEnumerable<ChatMessage>, ChatOptions?, string>? PromptFormatter { get; set; }
 
     /// <summary>Gets or sets whether to cache the most recent conversation.</summary>
     /// <remarks>

test/csharp/TestOnnxRuntimeGenAIAPI.cs

@@ -360,7 +360,7 @@ public void TestTopKTopPSearch()
         [IgnoreOnModelAbsenceFact(DisplayName = "TestChatClient")]
         public async Task TestChatClient()
         {
-            OnnxRuntimeGenAIChatClientOptions config = new()
+            OnnxRuntimeGenAIChatClientOptions options = new()
             {
                 StopSequences = ["<|system|>", "<|user|>", "<|assistant|>", "<|end|>"],
                 PromptFormatter = static (messages, options) =>
@@ -375,7 +375,7 @@ public async Task TestChatClient()
                 },
             };
 
-            using var client = new OnnxRuntimeGenAIChatClient(config, _phi2Path);
+            using var client = new OnnxRuntimeGenAIChatClient(_phi2Path, options);
 
             var completion = await client.GetResponseAsync("The quick brown fox jumps over the lazy dog.", new()
             {
@@ -384,7 +384,7 @@ public async Task TestChatClient()
                 StopSequences = ["."],
             });
 
-            Assert.NotEmpty(completion.Message.Text);
+            Assert.NotEmpty(completion.Text);
         }
 
         [IgnoreOnModelAbsenceFact(DisplayName = "TestTokenizerBatchEncodeDecode")]