Merge pull request #701 from TousakaRin/circuit_breaker

CircuitBreaker： fix race condition， adjust reset policy

src/brpc/circuit_breaker.cpp

@@ -99,10 +99,12 @@ bool CircuitBreaker::EmaErrorRecorder::OnCallEnd(int error_code,
 }
 
 void CircuitBreaker::EmaErrorRecorder::Reset() {
+    if (_sample_count_when_initializing.load(butil::memory_order_relaxed) < _window_size) {
         _sample_count_when_initializing.store(0, butil::memory_order_relaxed);
         _error_count_when_initializing.store(0, butil::memory_order_relaxed);
-    _ema_error_cost.store(0, butil::memory_order_relaxed);
         _ema_latency.store(0, butil::memory_order_relaxed);
+    }
+    _ema_error_cost.store(0, butil::memory_order_relaxed);
 }
 
 int64_t CircuitBreaker::EmaErrorRecorder::UpdateLatency(int64_t latency) {
@@ -162,7 +164,7 @@ CircuitBreaker::CircuitBreaker()
                    FLAGS_circuit_breaker_long_window_error_percent)
     , _short_window(FLAGS_circuit_breaker_short_window_size,
                     FLAGS_circuit_breaker_short_window_error_percent)
-    , _last_reset_time_ms(butil::cpuwide_time_ms())
+    , _last_reset_time_ms(0)
     , _isolation_duration_ms(FLAGS_circuit_breaker_min_isolation_duration_ms)
     , _isolated_times(0)
     , _broken(false) {

src/brpc/socket.cpp

@@ -728,6 +728,12 @@ int Socket::WaitAndReset(int32_t expected_nref) {
             _pipeline_q->clear();
         }
     }
+
+    SharedPart* sp = GetSharedPart();
+    if (sp) {
+        sp->circuit_breaker.Reset();
+        sp->recent_error_count.store(0, butil::memory_order_relaxed);
+    }
     return 0;
 }
 
@@ -750,11 +756,6 @@ void Socket::Revive() {
                 vref, MakeVRef(id_ver, nref + 1/*note*/),
                 butil::memory_order_release,
                 butil::memory_order_relaxed)) {
-            SharedPart* sp = GetSharedPart();
-            if (sp) {
-                sp->circuit_breaker.Reset();
-                sp->recent_error_count.store(0, butil::memory_order_relaxed);
-            }
             // Set this flag to true since we add additional ref again
             _recycle_flag.store(false, butil::memory_order_relaxed);
             if (_user) {

test/brpc_circuit_breaker_unittest.cpp

@@ -22,8 +22,8 @@ const int kShortWindowSize = 500;
 const int kLongWindowSize = 1000;
 const int kShortWindowErrorPercent = 10;
 const int kLongWindowErrorPercent = 5;
-const int kMinIsolationDurationMs = 100;
-const int kMaxIsolationDurationMs = 1000;
+const int kMinIsolationDurationMs = 10;
+const int kMaxIsolationDurationMs = 200;
 const int kErrorCodeForFailed = 131;
 const int kErrorCodeForSucc = 0;
 const int kErrorCost = 1000;
@@ -110,7 +110,7 @@ protected:
             FeedbackControl* fc =
                 new FeedbackControl(2 * kLongWindowSize, error_percent, &_circuit_breaker);
             fc_list->emplace_back(fc);
-            pthread_create(&tid, NULL, feed_back_thread, fc);
+            pthread_create(&tid, nullptr, feed_back_thread, fc);
             thread_list->push_back(tid);
         }
     }
@@ -123,8 +123,8 @@ TEST_F(CircuitBreakerTest, should_not_isolate) {
     std::vector<std::unique_ptr<FeedbackControl>> fc_list;
     StartFeedbackThread(&thread_list, &fc_list, 3);
     for (int  i = 0; i < kThreadNum; ++i) {
-        void* ret_data = NULL;
-        EXPECT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
         FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
         EXPECT_EQ(fc->_unhealthy_cnt, 0);
         EXPECT_TRUE(fc->_healthy);
@@ -136,22 +136,33 @@ TEST_F(CircuitBreakerTest, should_isolate) {
     std::vector<std::unique_ptr<FeedbackControl>> fc_list;
     StartFeedbackThread(&thread_list, &fc_list, 50);
     for (int  i = 0; i < kThreadNum; ++i) {
-        void* ret_data = NULL;
-        EXPECT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
         FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
         EXPECT_GT(fc->_unhealthy_cnt, 0);
         EXPECT_FALSE(fc->_healthy);
     }
 }
 
-TEST_F(CircuitBreakerTest, isolation_duration_grow) {
-    _circuit_breaker.Reset();
+TEST_F(CircuitBreakerTest, isolation_duration_grow_and_reset) {
     std::vector<pthread_t> thread_list;
     std::vector<std::unique_ptr<FeedbackControl>> fc_list;
     StartFeedbackThread(&thread_list, &fc_list, 100);
     for (int  i = 0; i < kThreadNum; ++i) {
-        void* ret_data = NULL;
-        EXPECT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
+        EXPECT_FALSE(fc->_healthy);
+        EXPECT_LE(fc->_healthy_cnt, kShortWindowSize);
+        EXPECT_GT(fc->_unhealthy_cnt, 0);
+    }
+    EXPECT_EQ(_circuit_breaker.isolation_duration_ms(), kMinIsolationDurationMs);
+
+    _circuit_breaker.Reset();
+    StartFeedbackThread(&thread_list, &fc_list, 100);
+    for (int  i = 0; i < kThreadNum; ++i) {
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
         FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
         EXPECT_FALSE(fc->_healthy);
         EXPECT_LE(fc->_healthy_cnt, kShortWindowSize);
@@ -160,11 +171,10 @@ TEST_F(CircuitBreakerTest, isolation_duration_grow) {
     EXPECT_EQ(_circuit_breaker.isolation_duration_ms(), kMinIsolationDurationMs * 2);
 
     _circuit_breaker.Reset();
-    bthread_usleep(kMinIsolationDurationMs * 1000);
     StartFeedbackThread(&thread_list, &fc_list, 100);
     for (int  i = 0; i < kThreadNum; ++i) {
-        void* ret_data = NULL;
-        EXPECT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
         FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
         EXPECT_FALSE(fc->_healthy);
         EXPECT_LE(fc->_healthy_cnt, kShortWindowSize);
@@ -173,15 +183,38 @@ TEST_F(CircuitBreakerTest, isolation_duration_grow) {
     EXPECT_EQ(_circuit_breaker.isolation_duration_ms(), kMinIsolationDurationMs * 4);
 
     _circuit_breaker.Reset();
-    bthread_usleep((kMaxIsolationDurationMs + kMinIsolationDurationMs) * 1000);
+    ::usleep((kMaxIsolationDurationMs + kMinIsolationDurationMs) * 1000);
     StartFeedbackThread(&thread_list, &fc_list, 100);
     for (int  i = 0; i < kThreadNum; ++i) {
-        void* ret_data = NULL;
-        EXPECT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
         FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
         EXPECT_FALSE(fc->_healthy);
         EXPECT_LE(fc->_healthy_cnt, kShortWindowSize);
         EXPECT_GT(fc->_unhealthy_cnt, 0);
     }
     EXPECT_EQ(_circuit_breaker.isolation_duration_ms(), kMinIsolationDurationMs);
+
+}
+
+TEST_F(CircuitBreakerTest, maximum_isolation_duration) {
+    brpc::FLAGS_circuit_breaker_max_isolation_duration_ms =
+        brpc::FLAGS_circuit_breaker_min_isolation_duration_ms + 1;
+    ASSERT_LT(brpc::FLAGS_circuit_breaker_max_isolation_duration_ms,
+              2 * brpc::FLAGS_circuit_breaker_min_isolation_duration_ms);
+    std::vector<pthread_t> thread_list;
+    std::vector<std::unique_ptr<FeedbackControl>> fc_list;
+
+    _circuit_breaker.Reset();
+    StartFeedbackThread(&thread_list, &fc_list, 100);
+    for (int  i = 0; i < kThreadNum; ++i) {
+        void* ret_data = nullptr;
+        ASSERT_EQ(pthread_join(thread_list[i], &ret_data), 0);
+        FeedbackControl* fc = static_cast<FeedbackControl*>(ret_data);
+        EXPECT_FALSE(fc->_healthy);
+        EXPECT_LE(fc->_healthy_cnt, kShortWindowSize);
+        EXPECT_GT(fc->_unhealthy_cnt, 0);
+    }
+    EXPECT_EQ(_circuit_breaker.isolation_duration_ms(),
+              brpc::FLAGS_circuit_breaker_max_isolation_duration_ms);
 }