Add extensible resilience pipelines

Author: ejsmith

global.json

@@ -1,7 +1,7 @@
 {
   "sdk": {
     "version": "8.0.100",
-    "rollForward": "latestMinor",
+    "rollForward": "latestMajor",
     "allowPrerelease": false
   }
 }

src/Foundatio.DataProtection/FoundatioStorageXmlRepository.cs

@@ -6,6 +6,7 @@
 using System.Xml.Linq;
 using Foundatio.Storage;
 using Foundatio.Utility;
+using Foundatio.Utility.Resilience;
 using Microsoft.AspNetCore.DataProtection.Repositories;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -22,17 +23,26 @@ public sealed class FoundatioStorageXmlRepository : IXmlRepository
 {
     private readonly IFileStorage _storage;
     private readonly ILogger _logger;
+    private readonly IResiliencePipeline _resiliencePipeline;
 
     /// <summary>
     /// Creates a new instance of the <see cref="FoundatioStorageXmlRepository"/>.
     /// </summary>
-    public FoundatioStorageXmlRepository(IFileStorage storage, ILoggerFactory loggerFactory = null)
+    public FoundatioStorageXmlRepository(IFileStorage storage, ILoggerFactory loggerFactory = null) : this(storage, null, loggerFactory)
+    {
+    }
+
+    /// <summary>
+    /// Creates a new instance of the <see cref="FoundatioStorageXmlRepository"/>.
+    /// </summary>
+    public FoundatioStorageXmlRepository(IFileStorage storage, IResiliencePipelineProvider resiliencePipelineProvider, ILoggerFactory loggerFactory = null)
     {
         if (storage == null)
             throw new ArgumentNullException(nameof(storage));
 
         _storage = new ScopedFileStorage(storage, "DataProtection");
         _logger = loggerFactory?.CreateLogger<FoundatioStorageXmlRepository>() ?? NullLogger<FoundatioStorageXmlRepository>.Instance;
+        _resiliencePipeline = resiliencePipelineProvider?.GetPipeline(nameof(FoundatioStorageXmlRepository)) ?? new FoundatioResiliencePipeline(TimeProvider.System, _logger);
     }
 
     /// <inheritdoc />
@@ -81,14 +91,14 @@ private Task StoreElementAsync(XElement element, string friendlyName)
         string path = String.Concat(!String.IsNullOrEmpty(friendlyName) ? friendlyName : Guid.NewGuid().ToString("N"), ".xml");
         _logger.LogTrace("Saving element: {File}.", path);
 
-        return Run.WithRetriesAsync(async () =>
+        return _resiliencePipeline.ExecuteAsync(async ct =>
         {
             using var memoryStream = new MemoryStream();
             element.Save(memoryStream, SaveOptions.DisableFormatting);
             memoryStream.Seek(0, SeekOrigin.Begin);
 
-            await _storage.SaveFileAsync(path, memoryStream).AnyContext();
+            await _storage.SaveFileAsync(path, memoryStream, ct).AnyContext();
             _logger.LogTrace("Saved element: {File}.", path);
-        });
+        }).AsTask();
     }
 }

src/Foundatio.TestHarness/Jobs/HelloWorldJob.cs

@@ -10,7 +10,7 @@ public class HelloWorldJob : JobBase
 {
     private readonly string _id;
 
-    public HelloWorldJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, loggerFactory)
+    public HelloWorldJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, null, loggerFactory)
     {
         _id = Guid.NewGuid().ToString("N").Substring(0, 10);
     }
@@ -35,7 +35,7 @@ public class FailingJob : JobBase
 
     public int RunCount { get; set; }
 
-    public FailingJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, loggerFactory)
+    public FailingJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, null, loggerFactory)
     {
         _id = Guid.NewGuid().ToString("N").Substring(0, 10);
     }
@@ -55,7 +55,7 @@ public class LongRunningJob : JobBase
     private readonly string _id;
     private int _iterationCount;
 
-    public LongRunningJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, loggerFactory)
+    public LongRunningJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, null, loggerFactory)
     {
         _id = Guid.NewGuid().ToString("N").Substring(0, 10);
     }

src/Foundatio.TestHarness/Jobs/JobQueueTestsBase.cs

@@ -108,7 +108,7 @@ await queue.EnqueueAsync(new SampleQueueWorkItem
             });
         });
 
-        var lockProvider = new ThrottlingLockProvider(new InMemoryCacheClient(o => o.LoggerFactory(Log)), allowedLockCount, TimeSpan.FromDays(1), null, Log);
+        var lockProvider = new ThrottlingLockProvider(new InMemoryCacheClient(o => o.LoggerFactory(Log)), allowedLockCount, TimeSpan.FromDays(1), null, null, Log);
         var job = new SampleQueueJobWithLocking(queue, lockProvider, null, Log);
         await Task.Delay(10);
         _logger.LogInformation("Starting RunUntilEmptyAsync");

src/Foundatio.TestHarness/Jobs/SampleQueueJob.cs

@@ -76,7 +76,7 @@ public class SampleQueueWorkItem
 
 public class SampleJob : JobBase
 {
-    public SampleJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, loggerFactory)
+    public SampleJob(TimeProvider timeProvider, ILoggerFactory loggerFactory) : base(timeProvider, null, loggerFactory)
     {
     }
 

src/Foundatio.TestHarness/Jobs/ThrottledJob.cs

@@ -12,7 +12,7 @@ public class ThrottledJob : JobWithLockBase
 {
     public ThrottledJob(ICacheClient client, ILoggerFactory loggerFactory = null) : base(loggerFactory)
     {
-        _locker = new ThrottlingLockProvider(client, 1, TimeSpan.FromMilliseconds(100), null, loggerFactory);
+        _locker = new ThrottlingLockProvider(client, 1, TimeSpan.FromMilliseconds(100), null, null, loggerFactory);
     }
 
     private readonly ILockProvider _locker;

src/Foundatio.TestHarness/Jobs/WithDependencyJob.cs

@@ -6,7 +6,7 @@ namespace Foundatio.Tests.Jobs;
 
 public class WithDependencyJob : JobBase
 {
-    public WithDependencyJob(MyDependency dependency, ILoggerFactory loggerFactory = null) : base(null, loggerFactory)
+    public WithDependencyJob(MyDependency dependency, ILoggerFactory loggerFactory = null) : base(null, null, loggerFactory)
     {
         Dependency = dependency;
     }

src/Foundatio.TestHarness/Jobs/WithLockingJob.cs

@@ -16,7 +16,7 @@ public class WithLockingJob : JobWithLockBase
 
     public WithLockingJob(ILoggerFactory loggerFactory) : base(loggerFactory)
     {
-        _locker = new CacheLockProvider(new InMemoryCacheClient(o => o.LoggerFactory(loggerFactory)), new InMemoryMessageBus(o => o.LoggerFactory(loggerFactory)), null, loggerFactory);
+        _locker = new CacheLockProvider(new InMemoryCacheClient(o => o.LoggerFactory(loggerFactory)), new InMemoryMessageBus(o => o.LoggerFactory(loggerFactory)), null, null, loggerFactory);
     }
 
     public int RunCount { get; set; }

src/Foundatio.TestHarness/Queue/QueueTestBase.cs

@@ -1281,7 +1281,7 @@ public virtual async Task CanDequeueWithLockingAsync()
         using var cache = new InMemoryCacheClient(o => o.LoggerFactory(Log));
         using var messageBus = new InMemoryMessageBus(o => o.LoggerFactory(Log));
 
-        var distributedLock = new CacheLockProvider(cache, messageBus, null, Log);
+        var distributedLock = new CacheLockProvider(cache, messageBus, null, null, Log);
         await CanDequeueWithLockingImpAsync(distributedLock);
     }
 
@@ -1335,7 +1335,7 @@ public virtual async Task CanHaveMultipleQueueInstancesWithLockingAsync()
         using var cache = new InMemoryCacheClient(o => o.LoggerFactory(Log));
         using var messageBus = new InMemoryMessageBus(o => o.LoggerFactory(Log));
 
-        var distributedLock = new CacheLockProvider(cache, messageBus, null, Log);
+        var distributedLock = new CacheLockProvider(cache, messageBus, null, null, Log);
         await CanHaveMultipleQueueInstancesWithLockingImplAsync(distributedLock);
     }
 

src/Foundatio/Caching/HybridCacheClient.cs

@@ -5,26 +5,33 @@
 using System.Threading.Tasks;
 using Foundatio.Messaging;
 using Foundatio.Utility;
+using Foundatio.Utility.Resilience;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 
 namespace Foundatio.Caching;
 
 public interface IHybridCacheClient : ICacheClient { }
 
-public class HybridCacheClient : IHybridCacheClient, IHaveTimeProvider, IHaveLogger
+public class HybridCacheClient : IHybridCacheClient, IHaveTimeProvider, IHaveLogger, IHaveLoggerFactory, IHaveResiliencePipelineProvider
 {
     protected readonly ICacheClient _distributedCache;
     protected readonly IMessageBus _messageBus;
     private readonly string _cacheId = Guid.NewGuid().ToString("N");
     private readonly InMemoryCacheClient _localCache;
     private readonly ILogger _logger;
+    private readonly ILoggerFactory _loggerFactory;
+    private readonly TimeProvider _timeProvider;
+    private readonly IResiliencePipelineProvider _resiliencePipelineProvider;
     private long _localCacheHits;
     private long _invalidateCacheCalls;
 
     public HybridCacheClient(ICacheClient distributedCacheClient, IMessageBus messageBus, InMemoryCacheClientOptions localCacheOptions = null, ILoggerFactory loggerFactory = null)
     {
-        _logger = loggerFactory?.CreateLogger<HybridCacheClient>() ?? NullLogger<HybridCacheClient>.Instance;
+        _loggerFactory = loggerFactory ?? distributedCacheClient.GetLoggerFactory() ?? localCacheOptions.LoggerFactory ?? NullLoggerFactory.Instance;
+        _logger = _loggerFactory.CreateLogger<HybridCacheClient>();
+        _timeProvider = distributedCacheClient.GetTimeProvider() ?? localCacheOptions?.TimeProvider ?? TimeProvider.System;
+        _resiliencePipelineProvider = distributedCacheClient.GetResiliencePipelineProvider() ?? localCacheOptions?.ResiliencePipelineProvider;
         _distributedCache = distributedCacheClient;
         _messageBus = messageBus;
         _messageBus.SubscribeAsync<InvalidateCache>(OnRemoteCacheItemExpiredAsync).AnyContext().GetAwaiter().GetResult();
@@ -39,7 +46,9 @@ public HybridCacheClient(ICacheClient distributedCacheClient, IMessageBus messag
     public long InvalidateCacheCalls => _invalidateCacheCalls;
 
     ILogger IHaveLogger.Logger => _logger;
-    TimeProvider IHaveTimeProvider.TimeProvider => _distributedCache.GetTimeProvider();
+    ILoggerFactory IHaveLoggerFactory.LoggerFactory => _loggerFactory;
+    TimeProvider IHaveTimeProvider.TimeProvider => _timeProvider;
+    IResiliencePipelineProvider IHaveResiliencePipelineProvider.ResiliencePipelineProvider => _resiliencePipelineProvider;
 
     private Task OnLocalCacheItemExpiredAsync(object sender, ItemExpiredEventArgs args)
     {