Skip to content

Commit 46bf417

Browse files
halspanghalspang
authored
Expose gRPC client retry options (#447)
* Expose gRPC client retry options Signed-off-by: Hal Spang <[email protected]>
1 parent f2dcc30 commit 46bf417

File tree

4 files changed

+168
-3
lines changed

4 files changed

+168
-3
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## (Unreleased)
44

5+
- Expose gRPC retry options in Azure Managed extensions ([#447](https://github.com/microsoft/durabletask-dotnet/pull/447))
6+
57
## v1.12.0
68

79
- Activity tag support ([#426](https://github.com/microsoft/durabletask-dotnet/pull/426))

‎src/Client/AzureManaged/DurableTaskSchedulerClientOptions.cs

Lines changed: 77 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@
66
using Azure.Identity;
77
using Grpc.Core;
88
using Grpc.Net.Client;
9-
using Microsoft.DurableTask;
109
using Microsoft.DurableTask.Client;
10+
using GrpcConfig = Grpc.Net.Client.Configuration;
1111

1212
namespace Microsoft.DurableTask;
1313

@@ -46,6 +46,11 @@ public class DurableTaskSchedulerClientOptions
4646
/// </summary>
4747
public bool AllowInsecureCredentials { get; set; }
4848

49+
/// <summary>
50+
/// Gets or sets the options that determine how and when calls made to the scheduler will be retried.
51+
/// </summary>
52+
public ClientRetryOptions? RetryOptions { get; set; }
53+
4954
/// <summary>
5055
/// Creates a new instance of <see cref="DurableTaskSchedulerClientOptions"/> from a connection string.
5156
/// </summary>
@@ -109,6 +114,45 @@ this.Credential is not null
109114
metadata.Add("Authorization", $"Bearer {token.Token}");
110115
});
111116

117+
GrpcConfig.ServiceConfig? serviceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig;
118+
if (this.RetryOptions != null)
119+
{
120+
GrpcConfig.RetryPolicy retryPolicy = new GrpcConfig.RetryPolicy
121+
{
122+
MaxAttempts = this.RetryOptions.MaxRetries ?? GrpcRetryPolicyDefaults.DefaultMaxAttempts,
123+
InitialBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.InitialBackoffMs ?? GrpcRetryPolicyDefaults.DefaultInitialBackoffMs),
124+
MaxBackoff = TimeSpan.FromMilliseconds(this.RetryOptions.MaxBackoffMs ?? GrpcRetryPolicyDefaults.DefaultMaxBackoffMs),
125+
BackoffMultiplier = this.RetryOptions.BackoffMultiplier ?? GrpcRetryPolicyDefaults.DefaultBackoffMultiplier,
126+
RetryableStatusCodes = { StatusCode.Unavailable }, // Always retry on Unavailable.
127+
};
128+
129+
if (this.RetryOptions.RetryableStatusCodes != null)
130+
{
131+
foreach (StatusCode statusCode in this.RetryOptions.RetryableStatusCodes)
132+
{
133+
// Added by default, don't need to have it added twice.
134+
if (statusCode == StatusCode.Unavailable)
135+
{
136+
continue;
137+
}
138+
139+
retryPolicy.RetryableStatusCodes.Add(statusCode);
140+
}
141+
}
142+
143+
GrpcConfig.MethodConfig methodConfig = new GrpcConfig.MethodConfig
144+
{
145+
// MethodName.Default applies this retry policy configuration to all gRPC methods on the channel.
146+
Names = { GrpcConfig.MethodName.Default },
147+
RetryPolicy = retryPolicy,
148+
};
149+
150+
serviceConfig = new GrpcConfig.ServiceConfig
151+
{
152+
MethodConfigs = { methodConfig },
153+
};
154+
}
155+
112156
// Production will use HTTPS, but local testing will use HTTP
113157
ChannelCredentials channelCreds = endpoint.StartsWith("https://", StringComparison.OrdinalIgnoreCase) ?
114158
ChannelCredentials.SecureSsl :
@@ -117,7 +161,7 @@ this.Credential is not null
117161
{
118162
Credentials = ChannelCredentials.Create(channelCreds, managedBackendCreds),
119163
UnsafeUseInsecureChannelCallCredentials = this.AllowInsecureCredentials,
120-
ServiceConfig = GrpcRetryPolicyDefaults.DefaultServiceConfig,
164+
ServiceConfig = serviceConfig,
121165
});
122166
}
123167

@@ -173,4 +217,35 @@ this.Credential is not null
173217
nameof(connectionString));
174218
}
175219
}
220+
221+
/// <summary>
222+
/// Options used to configure retries used when making calls to the Scheduler.
223+
/// </summary>
224+
public class ClientRetryOptions
225+
{
226+
/// <summary>
227+
/// Gets or sets the maximum number of times a call should be retried.
228+
/// </summary>
229+
public int? MaxRetries { get; set; }
230+
231+
/// <summary>
232+
/// Gets or sets the initial backoff in milliseconds.
233+
/// </summary>
234+
public int? InitialBackoffMs { get; set; }
235+
236+
/// <summary>
237+
/// Gets or sets the maximum backoff in milliseconds.
238+
/// </summary>
239+
public int? MaxBackoffMs { get; set; }
240+
241+
/// <summary>
242+
/// Gets or sets the backoff multiplier for exponential backoff.
243+
/// </summary>
244+
public double? BackoffMultiplier { get; set; }
245+
246+
/// <summary>
247+
/// Gets or sets the list of status codes that can be retried.
248+
/// </summary>
249+
public IList<StatusCode>? RetryableStatusCodes { get; set; }
250+
}
176251
}

‎test/Client/AzureManaged.Tests/Client.AzureManaged.Tests.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
<ItemGroup>
1212
<PackageReference Include="Microsoft.Extensions.DependencyInjection" />
13+
<PackageReference Include="Grpc.Core" />
1314
</ItemGroup>
1415

1516
<ItemGroup>

‎test/Client/AzureManaged.Tests/DurableTaskSchedulerClientExtensionsTests.cs

Lines changed: 88 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
using Azure.Core;
55
using Azure.Identity;
66
using FluentAssertions;
7+
using Grpc.Core;
78
using Microsoft.DurableTask.Client.Grpc;
89
using Microsoft.Extensions.DependencyInjection;
910
using Microsoft.Extensions.Options;
@@ -109,7 +110,7 @@ public void UseDurableTaskScheduler_WithNullParameters_ShouldThrowOptionsValidat
109110
// Assert
110111
var action = () => provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
111112
action.Should().Throw<OptionsValidationException>()
112-
.WithMessage(endpoint == null
113+
.WithMessage(endpoint == null
113114
? "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'EndpointAddress' with the error: 'Endpoint address is required'."
114115
: "DataAnnotation validation failed for 'DurableTaskSchedulerClientOptions' members: 'TaskHubName' with the error: 'Task hub name is required'.");
115116
}
@@ -193,4 +194,90 @@ public void UseDurableTaskScheduler_WithNamedOptions_ShouldConfigureCorrectly()
193194
options.ResourceId.Should().Be("https://durabletask.io");
194195
options.AllowInsecureCredentials.Should().BeFalse();
195196
}
197+
198+
[Fact]
199+
public void UseDurableTaskScheduler_WithEndpointAndCredentialAndRetryOptions_ShouldConfigureCorrectly()
200+
{
201+
// Arrange
202+
ServiceCollection services = new ServiceCollection();
203+
Mock<IDurableTaskClientBuilder> mockBuilder = new Mock<IDurableTaskClientBuilder>();
204+
mockBuilder.Setup(b => b.Services).Returns(services);
205+
DefaultAzureCredential credential = new DefaultAzureCredential();
206+
207+
// Act
208+
mockBuilder.Object.UseDurableTaskScheduler(ValidEndpoint, ValidTaskHub, credential, options =>
209+
options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
210+
{
211+
MaxRetries = 5,
212+
InitialBackoffMs = 100,
213+
MaxBackoffMs = 1000,
214+
BackoffMultiplier = 2.0,
215+
RetryableStatusCodes = new List<StatusCode> { StatusCode.Unknown }
216+
}
217+
);
218+
219+
// Assert
220+
ServiceProvider provider = services.BuildServiceProvider();
221+
IOptions<GrpcDurableTaskClientOptions>? options = provider.GetService<IOptions<GrpcDurableTaskClientOptions>>();
222+
options.Should().NotBeNull();
223+
224+
// Validate the configured options
225+
DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
226+
clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
227+
clientOptions.TaskHubName.Should().Be(ValidTaskHub);
228+
clientOptions.Credential.Should().BeOfType<DefaultAzureCredential>();
229+
clientOptions.RetryOptions.Should().NotBeNull();
230+
// The assert not null doesn't clear the syntax warning about null checks.
231+
if (clientOptions.RetryOptions != null)
232+
{
233+
clientOptions.RetryOptions.MaxRetries.Should().Be(5);
234+
clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
235+
clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
236+
clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
237+
clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
238+
}
239+
}
240+
241+
[Fact]
242+
public void UseDurableTaskScheduler_WithConnectionStringAndRetryOptions_ShouldConfigureCorrectly()
243+
{
244+
// Arrange
245+
ServiceCollection services = new ServiceCollection();
246+
Mock<IDurableTaskClientBuilder> mockBuilder = new Mock<IDurableTaskClientBuilder>();
247+
mockBuilder.Setup(b => b.Services).Returns(services);
248+
string connectionString = $"Endpoint={ValidEndpoint};Authentication=DefaultAzure;TaskHub={ValidTaskHub}";
249+
250+
// Act
251+
mockBuilder.Object.UseDurableTaskScheduler(connectionString, options =>
252+
options.RetryOptions = new DurableTaskSchedulerClientOptions.ClientRetryOptions
253+
{
254+
MaxRetries = 5,
255+
InitialBackoffMs = 100,
256+
MaxBackoffMs = 1000,
257+
BackoffMultiplier = 2.0,
258+
RetryableStatusCodes = new List<StatusCode> { StatusCode.Unknown }
259+
}
260+
);
261+
262+
// Assert
263+
ServiceProvider provider = services.BuildServiceProvider();
264+
IOptions<GrpcDurableTaskClientOptions>? options = provider.GetService<IOptions<GrpcDurableTaskClientOptions>>();
265+
options.Should().NotBeNull();
266+
267+
// Validate the configured options
268+
DurableTaskSchedulerClientOptions clientOptions = provider.GetRequiredService<IOptions<DurableTaskSchedulerClientOptions>>().Value;
269+
clientOptions.EndpointAddress.Should().Be(ValidEndpoint);
270+
clientOptions.TaskHubName.Should().Be(ValidTaskHub);
271+
clientOptions.Credential.Should().BeOfType<DefaultAzureCredential>();
272+
clientOptions.RetryOptions.Should().NotBeNull();
273+
// The assert not null doesn't clear the syntax warning about null checks.
274+
if (clientOptions.RetryOptions != null)
275+
{
276+
clientOptions.RetryOptions.MaxRetries.Should().Be(5);
277+
clientOptions.RetryOptions.InitialBackoffMs.Should().Be(100);
278+
clientOptions.RetryOptions.MaxBackoffMs.Should().Be(1000);
279+
clientOptions.RetryOptions.BackoffMultiplier.Should().Be(2.0);
280+
clientOptions.RetryOptions.RetryableStatusCodes.Should().Contain(StatusCode.Unknown);
281+
}
282+
}
196283
}

0 commit comments

Comments
 (0)