fix data race for circuit breaker

src/brpc/circuit_breaker.cpp

@@ -99,10 +99,12 @@ bool CircuitBreaker::EmaErrorRecorder::OnCallEnd(int error_code,
 }
 
 void CircuitBreaker::EmaErrorRecorder::Reset() {
+    if (_sample_count_when_initializing.load(butil::memory_order_relaxed) < _window_size) {
         _sample_count_when_initializing.store(0, butil::memory_order_relaxed);
         _error_count_when_initializing.store(0, butil::memory_order_relaxed);
-    _ema_error_cost.store(0, butil::memory_order_relaxed);
         _ema_latency.store(0, butil::memory_order_relaxed);
+    }
+    _ema_error_cost.store(0, butil::memory_order_relaxed);
 }
 
 int64_t CircuitBreaker::EmaErrorRecorder::UpdateLatency(int64_t latency) {
@@ -162,9 +164,10 @@ CircuitBreaker::CircuitBreaker()
                    FLAGS_circuit_breaker_long_window_error_percent)
     , _short_window(FLAGS_circuit_breaker_short_window_size,
                     FLAGS_circuit_breaker_short_window_error_percent)
-    , _last_reset_time_ms(butil::cpuwide_time_ms())
+    , _last_revived_time_ms(butil::cpuwide_time_ms())
     , _isolation_duration_ms(FLAGS_circuit_breaker_min_isolation_duration_ms)
     , _isolated_times(0)
+    , _is_first_call_after_revived(true)
     , _broken(false) {
 }
 
@@ -172,6 +175,10 @@ bool CircuitBreaker::OnCallEnd(int error_code, int64_t latency) {
     if (_broken.load(butil::memory_order_relaxed)) {
         return false;
     }
+    if (_is_first_call_after_revived.load(butil::memory_order_relaxed) &&
+        _is_first_call_after_revived.exchange(false, butil::memory_order_relaxed)) {
+      _last_revived_time_ms.store(butil::cpuwide_time_ms(), butil::memory_order_relaxed);
+    }
     if (_long_window.OnCallEnd(error_code, latency) &&
         _short_window.OnCallEnd(error_code, latency)) {
         return true;
@@ -183,7 +190,8 @@ bool CircuitBreaker::OnCallEnd(int error_code, int64_t latency) {
 void CircuitBreaker::Reset() {
     _long_window.Reset();
     _short_window.Reset();
-    _last_reset_time_ms = butil::cpuwide_time_ms();
+    _last_revived_time_ms.store(butil::cpuwide_time_ms(), butil::memory_order_relaxed);
+    _is_first_call_after_revived.store(true, butil::memory_order_relaxed);
     _broken.store(false, butil::memory_order_release);
 }
 
@@ -201,7 +209,7 @@ void CircuitBreaker::UpdateIsolationDuration() {
         FLAGS_circuit_breaker_max_isolation_duration_ms;
     const int min_isolation_duration_ms =
         FLAGS_circuit_breaker_min_isolation_duration_ms;
-    if (now_time_ms - _last_reset_time_ms < max_isolation_duration_ms) {
+    if (now_time_ms - _last_revived_time_ms < max_isolation_duration_ms) {
         isolation_duration_ms =
             std::min(isolation_duration_ms * 2, max_isolation_duration_ms);
     } else {

src/brpc/circuit_breaker.h

@@ -82,9 +82,10 @@ private:
 
     EmaErrorRecorder _long_window;
     EmaErrorRecorder _short_window;
-    int64_t _last_reset_time_ms; 
+    butil::atomic<int64_t> _last_revived_time_ms;
     butil::atomic<int> _isolation_duration_ms;
     butil::atomic<int> _isolated_times;
+    butil::atomic<bool> _is_first_call_after_revived;
     butil::atomic<bool> _broken;
 };
 

src/brpc/socket.cpp

@@ -728,6 +728,12 @@ int Socket::WaitAndReset(int32_t expected_nref) {
             _pipeline_q->clear();
         }
     }
+
+    SharedPart* sp = GetSharedPart();
+    if (sp) {
+        sp->circuit_breaker.Reset();
+        sp->recent_error_count.store(0, butil::memory_order_relaxed);
+    }
     return 0;
 }
 
@@ -750,11 +756,6 @@ void Socket::Revive() {
                 vref, MakeVRef(id_ver, nref + 1/*note*/),
                 butil::memory_order_release,
                 butil::memory_order_relaxed)) {
-            SharedPart* sp = GetSharedPart();
-            if (sp) {
-                sp->circuit_breaker.Reset();
-                sp->recent_error_count.store(0, butil::memory_order_relaxed);
-            }
             // Set this flag to true since we add additional ref again
             _recycle_flag.store(false, butil::memory_order_relaxed);
             if (_user) {