feat: Retry failed Event requests

src/PostHog/Api/PostHogApiClient.cs

@@ -4,6 +4,7 @@
 using System.Text.Json;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Polly;
 using PostHog.Json;
 using PostHog.Library;
 using PostHog.Versioning;
@@ -21,6 +22,7 @@ internal sealed class PostHogApiClient : IDisposable
 
     readonly TimeProvider _timeProvider;
     readonly HttpClient _httpClient;
+    readonly ResiliencePipeline _resiliencePipeline;
     readonly IOptions<PostHogOptions> _options;
     readonly ILogger<PostHogApiClient> _logger;
 
@@ -30,11 +32,13 @@ internal sealed class PostHogApiClient : IDisposable
     /// <param name="httpClient">The <see cref="HttpClient"/> used to make requests.</param>
     /// <param name="options">The options used to configure this client.</param>
     /// <param name="timeProvider">The time provider <see cref="TimeProvider"/> to use to determine time.</param>
+    /// <param name="resiliencePipeline">The <see cref="ResiliencePipeline"/> to use for event request retries and timeouts.</param>
     /// <param name="logger">The logger.</param>
     public PostHogApiClient(
         HttpClient httpClient,
         IOptions<PostHogOptions> options,
         TimeProvider timeProvider,
+        ResiliencePipeline resiliencePipeline,
         ILogger<PostHogApiClient> logger)
     {
         _options = options;
@@ -47,6 +51,7 @@ public PostHogApiClient(
         var arch = RuntimeInformation.ProcessArchitecture;
         _httpClient.DefaultRequestHeaders.UserAgent.ParseAdd($"{LibraryName}/{VersionConstants.Version} ({framework}; {os}; {arch})");
 
+        _resiliencePipeline = resiliencePipeline;
         logger.LogTraceApiClientCreated(HostUrl);
         _logger = logger;
     }
@@ -74,8 +79,13 @@ public async Task<ApiResult> CaptureBatchAsync(
             ["batch"] = events.ToReadOnlyList()
         };
 
-        return await _httpClient.PostJsonAsync<ApiResult>(endpointUrl, payload, cancellationToken)
-               ?? new ApiResult(0);
+        return await _resiliencePipeline.ExecuteAsync(
+            static async (state, token) => await state.Client.PostJsonAsync<ApiResult>(
+                state.Endpoint,
+                state.Payload,
+                token) ?? new ApiResult(0),
+            (Endpoint: endpointUrl, Payload: payload, Client: _httpClient),
+            cancellationToken);
     }
 
     /// <summary>
@@ -90,8 +100,13 @@ public async Task<ApiResult> SendEventAsync(
 
         var endpointUrl = new Uri(HostUrl, "capture");
 
-        return await _httpClient.PostJsonAsync<ApiResult>(endpointUrl, payload, cancellationToken)
-               ?? new ApiResult(0);
+        return await _resiliencePipeline.ExecuteAsync(
+            static async (state, token) => await state.Client.PostJsonAsync<ApiResult>(
+                state.Endpoint,
+                state.Payload,
+                token) ?? new ApiResult(0),
+            (Endpoint: endpointUrl, Payload: payload, Client: _httpClient),
+            cancellationToken);
     }
 
     /// <summary>

src/PostHog/Config/PostHogOptions.cs

@@ -33,6 +33,26 @@ public sealed class PostHogOptions : IOptions<PostHogOptions>
     /// </summary>
     public Uri HostUrl { get; set; } = new("https://us.i.posthog.com");
 
+    /// <summary>
+    /// Timeout in milliseconds for any calls. Defaults to 10 seconds.
+    /// </summary>
+    public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(10);
+
+    /// <summary>
+    /// Number of times to retry a failed request. Defaults to 3.
+    /// </summary>
+    public int MaxRetries { get; set; } = 3;
+
+    /// <summary>
+    /// Base delay between retries. Defaults to 3 seconds.
+    /// </summary>
+    public TimeSpan RetryDelay { get; set; } = TimeSpan.FromSeconds(3);
+
+    /// <summary>
+    /// Maximum delay between retries. Defaults to 10 seconds.
+    /// </summary>
+    public TimeSpan MaxRetryDelay { get; set; } = TimeSpan.FromSeconds(10);
+
     /// <summary>
     /// Default properties to send when capturing events. These properties override any properties with the same
     /// key sent with the event.
@@ -96,4 +116,4 @@ public sealed class PostHogOptions : IOptions<PostHogOptions>
     // Explicit implementation to hide this value from most users.
     // This is here to make it easier to instantiate the client with the options.
     PostHogOptions IOptions<PostHogOptions>.Value => this;
-}
+}

src/PostHog/PostHog.csproj

@@ -8,6 +8,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+        <PackageReference Include="Polly.Core" Version="8.6.4" />
         <PackageReference Include="System.Text.Json" Version="8.0.5" />
         <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" />
         <PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.1" />

src/PostHog/PostHogClient.cs

@@ -3,6 +3,9 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
+using Polly;
+using Polly.Retry;
+using Polly.Timeout;
 using PostHog.Api;
 using PostHog.ErrorTracking;
 using PostHog.Exceptions;
@@ -35,13 +38,15 @@ public sealed class PostHogClient : IPostHogClient
     /// <param name="httpClientFactory">Creates <see cref="HttpClient"/> for making requests to PostHog's API.</param>
     /// <param name="taskScheduler">Used to run tasks on the background.</param>
     /// <param name="timeProvider">The time provider <see cref="TimeProvider"/> to use to determine time.</param>
+    /// <param name="resiliencePipeline">The <see cref="Polly.ResiliencePipeline"/> to use when making event requests to PostHog's API.</param>
     /// <param name="loggerFactory">The logger factory.</param>
     public PostHogClient(
         IOptions<PostHogOptions> options,
         IFeatureFlagCache? featureFlagsCache = null,
         IHttpClientFactory? httpClientFactory = null,
         ITaskScheduler? taskScheduler = null,
         TimeProvider? timeProvider = null,
+        ResiliencePipeline? resiliencePipeline = null,
         ILoggerFactory? loggerFactory = null)
     {
         _options = NotNull(options);
@@ -51,10 +56,31 @@ public PostHogClient(
         _timeProvider = timeProvider ?? TimeProvider.System;
         loggerFactory ??= NullLoggerFactory.Instance;
 
+        resiliencePipeline ??= new ResiliencePipelineBuilder
+            {
+                TimeProvider = _timeProvider
+            }
+            .AddRetry(new RetryStrategyOptions
+            {
+                MaxRetryAttempts = _options.Value.MaxRetries,
+                Delay = _options.Value.RetryDelay,
+                MaxDelay = _options.Value.MaxRetryDelay,
+                BackoffType = DelayBackoffType.Exponential,
+                ShouldHandle = args => args.Outcome switch
+                {
+                    { Exception: ApiException } => PredicateResult.True(),
+                    { Exception: TimeoutRejectedException } => PredicateResult.True(),
+                    _ => PredicateResult.False()
+                }
+            })
+            .AddTimeout(_options.Value.RequestTimeout)
+            .Build();
+
         _apiClient = new PostHogApiClient(
             httpClientFactory.CreateClient(nameof(PostHogClient)),
             options,
             _timeProvider,
+            resiliencePipeline,
             loggerFactory.CreateLogger<PostHogApiClient>()
         );
         _asyncBatchHandler = new AsyncBatchHandler<CapturedEvent, CapturedEventBatchContext>(

tests/TestLibrary/Fakes/FakeHttpMessageHandlerExtensions.cs

@@ -21,6 +21,14 @@ public static FakeHttpMessageHandler.RequestHandler AddBatchResponse(this FakeHt
             HttpMethod.Post,
             responseBody: new { status = 1 });
 
+    public static FakeHttpMessageHandler.RequestHandler AddBatchResponseException(
+        this FakeHttpMessageHandler handler,
+        Exception exception)
+        => handler.AddResponseException(
+            new Uri("https://us.i.posthog.com/batch"),
+            HttpMethod.Post,
+            exception);
+
     public static FakeHttpMessageHandler.RequestHandler AddDecideResponseException(
         this FakeHttpMessageHandler handler,
         Exception exception)

tests/TestLibrary/TestContainer.cs

@@ -3,6 +3,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Time.Testing;
+using Polly;
 using PostHog;
 using PostHog.Config;
 using PostHog.Library;
@@ -97,7 +98,7 @@ void ConfigureServices(IServiceCollection services)
             var taskScheduler = sp.GetRequiredService<ITaskScheduler>();
             var timeProvider = sp.GetRequiredService<TimeProvider>();
             var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
-            return new PostHogClient(options, cache, httpClientFactory, taskScheduler, timeProvider, loggerFactory);
+            return new PostHogClient(options, cache, httpClientFactory, taskScheduler, timeProvider, ResiliencePipeline.Empty, loggerFactory);
         });
     }
 

tests/UnitTests/PostHogClientTests.cs

@@ -1,17 +1,21 @@
-using System.Reflection;
-using System.Runtime.InteropServices;
-using System.Runtime.Loader;
-using System.Text;
-using System.Text.Json;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.Emit;
 using Microsoft.CodeAnalysis.Text;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Polly;
+using Polly.Retry;
+using Polly.Timeout;
 using PostHog;
+using PostHog.Library;
 using PostHog.Versioning;
+using System.Reflection;
+using System.Runtime.InteropServices;
+using System.Runtime.Loader;
+using System.Text;
+using System.Text.Json;
 using UnitTests.Fakes;
 
 #pragma warning disable CA2000
@@ -663,6 +667,86 @@ public async Task CaptureWithTimestampParameterOverridesTimestampInProperties()
                      }
                      """, received);
     }
+
+    // Polly timeout, retry, and backoff flow when capturing events
+    [Fact]
+    public async Task CaptureRetriesOnErrorWithDefaultPipeline()
+    {
+        var container = new TestContainer();
+        var fakeTime = container.FakeTimeProvider;
+        var observed = new List<(int Attempt, TimeSpan Delay)>();
+
+        var resilienceBuilder = new ResiliencePipelineBuilder
+        {
+            TimeProvider = container.FakeTimeProvider
+        };
+
+        // Resembles the default resilience pipeline instead of the empty one usually used for tests
+        // We also capture the retry attempts and delays for verification with OnRetry
+        var pipeline = resilienceBuilder
+            .AddRetry(new RetryStrategyOptions
+            {
+                MaxRetryAttempts = 3,
+                Delay = TimeSpan.FromSeconds(3),
+                MaxDelay = TimeSpan.FromSeconds(10),
+                BackoffType = DelayBackoffType.Exponential,
+                ShouldHandle = args => args.Outcome switch
+                {
+                    { Exception: ApiException } => PredicateResult.True(),
+                    { Exception: TimeoutRejectedException } => PredicateResult.True(),
+                    _ => PredicateResult.False()
+                },
+                OnRetry = args =>
+                {
+                    observed.Add((args.AttemptNumber, args.RetryDelay));
+                    return default;
+                }
+            })
+            .AddTimeout(TimeSpan.FromSeconds(15))
+            .Build();
+
+        var client = container.Activate<PostHogClient>(pipeline);
+
+        container.FakeHttpMessageHandler.AddBatchResponseException(new ApiException("Error"));
+        container.FakeHttpMessageHandler.AddBatchResponseException(new ApiException("Error Retry 1"));
+        container.FakeHttpMessageHandler.AddBatchResponseException(new ApiException("Error Retry 2"));
+        container.FakeHttpMessageHandler.AddBatchResponseException(new ApiException("Error Retry 3"));
+
+        client.Capture("test-user", "retry-event");
+        var flushTask = client.FlushAsync();
+        await Task.Yield();
+
+        // Advance fake time to fire the scheduled retries immediately
+        fakeTime.Advance(TimeSpan.FromSeconds(3));
+        fakeTime.Advance(TimeSpan.FromSeconds(6));
+        fakeTime.Advance(TimeSpan.FromSeconds(12));
+
+        // After all the retries we should get the last exception
+        var lastException = await Assert.ThrowsAsync<ApiException>(() => flushTask);
+        Assert.Equal("Error Retry 3", lastException.Message);
+        Assert.Collection(observed,
+            x => { Assert.Equal(0, x.Attempt); Assert.Equal(TimeSpan.FromSeconds(3), x.Delay); },
+            x => { Assert.Equal(1, x.Attempt); Assert.Equal(TimeSpan.FromSeconds(6), x.Delay); },
+            // Due to delay cap it should be 10 not 12 seconds
+            x => { Assert.Equal(2, x.Attempt); Assert.Equal(TimeSpan.FromSeconds(10), x.Delay); }
+        );
+
+        client.Capture("test-user", "successful-retry-event");
+        container.FakeHttpMessageHandler.AddBatchResponseException(new TimeoutRejectedException());
+        var requestHandler = container.FakeHttpMessageHandler.AddBatchResponse();
+
+        var flushTask2 = client.FlushAsync();
+        await Task.Yield();
+
+        // Advance fake time past timeout to trigger the retry
+        fakeTime.Advance(TimeSpan.FromSeconds(16));
+        await flushTask2;
+
+        // After timeout the retried request should succeed
+        Assert.Equal(TimeSpan.FromSeconds(3), observed.Last().Delay);
+        var received = requestHandler.GetReceivedRequestBody(indented: true);
+        Assert.Contains("successful-retry-event", received, StringComparison.OrdinalIgnoreCase);
+    }
 }
 
 public class TheCaptureExceptionMethod
@@ -896,9 +980,7 @@ public async Task CaptureExceptionCauseIOFailureEmptyContext()
         }
     }
 
-    // This test is pretty expensive because it dynamically compiles and loads an assembly.
-    // Consider alternatives.
-    [Fact]
+    [Fact(Skip= "Takes long time to run because it dynamically compiles and loads an assembly.")]
     public async Task CaptureExceptionWithInvalidFilePathInStackFrame()
     {
         var (_, requestHandler, client) = CreateClient();