Circuit Breaker Pattern

 

One of the big differences between in-memory calls and remote calls is that remote calls can fail or hang without a response until some timeout limit is reached. 

The basic idea behind the circuit breaker pattern is this: wrap a protected function call in a circuit breaker object, which monitors for failures. Once the failures reach a certain threshold, the circuit breaker trips, and all further calls to the circuit breaker return with an error or with some alternative service or default message, without the protected call being made at all. This will make sure system is responsive and threads are not waiting for an unresponsive call. 

The circuit breaker has three distinct states:

• Closed When everything is normal, all calls pass through to the services. When the number of failures exceeds a threshold the breaker trips, and it goes into the Open state.
• Open The circuit breaker returns an error for calls without executing the function.
• Half-Open After a timeout period, the circuit switches to a half-open state to test if the underlying problem still exists. If a single call fails in this half-open state, the breaker is once again tripped. If it succeeds, the circuit breaker resets back to the normal, closed state.

Example Code (C# .NET Core Console Application)

using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using System;

namespace CircuitBreakerPattern
{
    public class CircuitBreaker
    {
        public enum State { Closed, Open, HalfOpen };

        public int Threshold { get; set; }
        public TimeSpan Timeout { get; set; }

        public bool Run(Action remoteCall)
        {
            bool runSucceeded = false;

            switch (state)
            {

                case State.Closed: // Normal operating state.

                    for (int i = 1; i <= Threshold; i++)
                    {
                        logger.LogInformation(
                            $"Circuit closed. Attempting call. Attempt # {i}");

                        try
                        {
                            remoteCall(); // Attempt remote call.
                            runSucceeded = true; // It worked.
                            break;
                        }
                        catch
                        {
                            runSucceeded = false; // It didn't work.
                        }
                    }
                    if (runSucceeded == false)
                    {
                        state = State.Open;
                        startTime = DateTime.Now;
                    }
                    break;

                case State.Open: // Circuit breaker was tripped.

                    if (DateTime.Now - startTime > Timeout)
                    {
                        logger.LogInformation(
                            $"Circuit open. Timeout ended. Changing to half open.");

                        // Important: in the Open state, DO NOT attempt remote call.
                        // Instead, make recursive call with changed state.
                        state = State.HalfOpen;
                        runSucceeded = Run(remoteCall); // recurse
                    }
                    else
                    {
                        runSucceeded = false; // Remote call was not attempted.
                    }

                    break;

                case State.HalfOpen: // Test if circuit has recovered.

                    logger.LogInformation(
                        $"Circuit half open. Attempting remote call.");

                    try
                    {
                        remoteCall(); // Attempt remote call.
                        runSucceeded = true; // It worked.
                        break;
                    }
                    catch
                    {
                        runSucceeded = false; // It didn't work.
                    }

                    if (runSucceeded == false)
                    {
                        state = State.Open;
                        startTime = DateTime.Now;
                    }
                    else
                    {
                        state = State.Closed;
                    }

                    break;

                default:
                    throw new Exception("invalid state");
            }

            logger.LogInformation(
                $"Run complete. Result = {runSucceeded}");

            return runSucceeded;
        }

        private DateTime startTime;
        private State state = State.Closed; // Initial state is Closed.
        private readonly ILogger logger = 
                new ServiceCollection()
                    .AddLogging(config => config.AddConsole()) 
                    .AddTransient<CircuitBreaker>()
                    .BuildServiceProvider()
                    .GetService<ILogger<CircuitBreaker>>();
    }

    // This simple console app can serve as a test program.
    // *** Note: MyRemoteServer is just a placeholder.

    public class Program
    {
        static void Main(string[] args)
        {
            var circuitBreaker = new CircuitBreaker() 
            {
                Threshold = 3, Timeout = new TimeSpan(0, 0, 2)
            };

            // *** Replace with your version of MyRemoteServer.
            var server = new MyRemoteServer();
            Action remoteCall = () => { server.Ping(); };

            try
            {
                var success = circuitBreaker.Run(remoteCall);
                if (!success)
                {
                    // Possible transient fault. Wait and try again.
                    System.Threading.Thread.Sleep(2500);
                    success = circuitBreaker.Run(remoteCall);
                }
            }
            catch (Exception e)
            {
                Console.WriteLine(e);
                throw;
            }        
        }
    }   

    // *** Your version of MyRemoteServer goes here.
    public class MyRemoteServer
    {

        public string Ping()
        {
            // throw new System.Exception(); // Simulate failed remote call.
            return "Pong"; // Simulate successful remote call.
        }
    }
}

// Copyright (c) 2021 Robi Indra Das
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

--

References:

https://martinfowler.com/bliki/CircuitBreaker.html

https://dzone.com/articles/circuit-breaker-pattern

https://docs.microsoft.com/en-us/azure/architecture/patterns/circuit-breaker

https://www.blinkingcaret.com/2018/02/14/net-core-console-logging/