Patch svn r35031

Change-Id: I8ebb509b4a07f543f86bf86de9dab57c29d64927

bthread/task_group.cpp

@@ -132,6 +132,10 @@ bool TaskGroup::wait_task(bthread_t* tid, size_t* seed, size_t offset) {
         if (rc <= 0) {
             return rc == 0;
         }
+        // When BTHREAD_FAIR_WSQ is defined, profiling shows that cpu cost of
+        // WSQ::steal() in example/multi_threaded_echo_c++ changes from 1.9%
+        // to 2.9%
+#ifndef BTHREAD_FAIR_WSQ
         if (_rq.volatile_size() != 0) {
             _rq_mutex.lock();
             const bool popped = _rq.pop(tid);
@@ -140,6 +144,11 @@ bool TaskGroup::wait_task(bthread_t* tid, size_t* seed, size_t offset) {
                 return true;
             }
         }
+#else
+        if (_rq.steal(tid)) {
+            return true;
+        }
+#endif
     } while (true);
 }
 
@@ -510,9 +519,13 @@ void TaskGroup::ending_sched(TaskGroup** pg) {
     TaskGroup* g = *pg;
     bthread_t next_tid = 0;
     // Find next task to run, if none, switch to idle thread of the group.
+#ifndef BTHREAD_FAIR_WSQ
     g->_rq_mutex.lock();
     const bool popped = g->_rq.pop(&next_tid);
     g->_rq_mutex.unlock();
+#else
+    const bool popped = g->_rq.steal(&next_tid);
+#endif
     if (!popped) {
         if (!g->_control->steal_task(
                 &next_tid, &g->_steal_seed, g->_steal_offset)) {
@@ -549,9 +562,13 @@ void TaskGroup::sched(TaskGroup** pg) {
     TaskGroup* g = *pg;
     bthread_t next_tid = 0;
     // Find next task to run, if none, switch to idle thread of the group.
+#ifndef BTHREAD_FAIR_WSQ
     g->_rq_mutex.lock();
     const bool popped = g->_rq.pop(&next_tid);
     g->_rq_mutex.unlock();
+#else
+    const bool popped = g->_rq.steal(&next_tid);
+#endif
     if (!popped) {
         if (!g->_control->steal_task(
                 &next_tid, &g->_steal_seed, g->_steal_offset)) {

bthread/task_group.h

@@ -182,12 +182,8 @@ private:
     StackContainer* _main_stack_container;
     bthread_t _main_tid;
     pthread_t _creation_pthread;
-    // NOTE: _rq_mutex is not in the same cacheline of _rq because in some
-    // scenarios _rq_mutex is mostly locked/unlocked by this thread only.
-    // As a contrast, _rq is always stolen by other threads frequently.
     base::Mutex _rq_mutex;
-
-    WorkStealingQueue<bthread_t> BAIDU_CACHELINE_ALIGNMENT _rq;
+    WorkStealingQueue<bthread_t> _rq;
 };
 
 }  // namespace bthread

bthread/work_stealing_queue.h

@@ -18,9 +18,9 @@ class WorkStealingQueue {
 public:
     WorkStealingQueue()
         : _bottom(1)
-        , _top(1)
         , _capacity(0)
-        , _buffer(NULL) {
+        , _buffer(NULL)
+        , _top(1) {
     }
 
     ~WorkStealingQueue() {
@@ -132,9 +132,9 @@ private:
     DISALLOW_COPY_AND_ASSIGN(WorkStealingQueue);
 
     base::atomic<size_t> _bottom;
-    base::atomic<size_t> _top;
     size_t _capacity;
     T* _buffer;
+    base::atomic<size_t> BAIDU_CACHELINE_ALIGNMENT _top;
 };
 
 }  // namespace bthread