Support gcc7 glib2.25 openssl 1.1 and fedora26

Change-Id: I05e1685765ed24345ff8af6cf3608d674fa5ab05

Makefile

+NEED_LIBPROTOC=1
 include config.mk
+
+# Notes on the flags:
+# 1. -fno-omit-frame-pointer is required by perf/tcmalloc-profiler which use frame pointers by default
+# 2. -D__const__= MUST be added in user's gcc compilation as well to avoid the over-optimization on TLS variables by gcc
+# 3. Removed -Werror to not block compilation for non-vital warnings, especially when the code is compiled on newer systems. If you use the code in production, add -Werror back
 CPPFLAGS=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D__const__= -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -DBRPC_REVISION=\"$(shell git rev-parse --short HEAD)\"
-#Add -fno-omit-frame-pointer: perf/tcmalloc-profiler uses frame pointers by default
-CXXFLAGS=$(CPPFLAGS) -O2 -g -pipe -Wall -W -Werror -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x -include brpc/config.h
-CFLAGS=$(CPPFLAGS) -O2 -g -pipe -Wall -W -Werror -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer
+CXXFLAGS=$(CPPFLAGS) -O2 -g -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x -include brpc/config.h
+CFLAGS=$(CPPFLAGS) -O2 -g -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer
 HDRPATHS=-I. $(addprefix -I, $(HDRS))
 LIBPATHS = $(addprefix -L, $(LIBS))
 SRCEXTS = .c .cc .cpp .proto
 HDREXTS = .h .hpp
-#dyanmic linking of libprotoc.so crashes on ubuntu when protoc-gen-mcpack is invoked
-STATIC_LINKINGS += -lprotoc
+
+ifeq ($(shell test $(shell $(CXX) -dumpversion) -ge 7; echo $$?),0)
+CXXFLAGS+=-Wno-aligned-new
+endif
 
 BASE_SOURCES = \
     base/third_party/dmg_fp/g_fmt.cc \

base/errno.cpp

@@ -15,13 +15,13 @@
 
 namespace base {
 
-static const int ERRNO_BEGIN = -32768;
-static const int ERRNO_END = 32768;
-const char* errno_desc[ERRNO_END - ERRNO_BEGIN] = {};
-pthread_mutex_t modify_desc_mutex = PTHREAD_MUTEX_INITIALIZER;
+const int ERRNO_BEGIN = -32768;
+const int ERRNO_END = 32768;
+static const char* errno_desc[ERRNO_END - ERRNO_BEGIN] = {};
+static pthread_mutex_t modify_desc_mutex = PTHREAD_MUTEX_INITIALIZER;
 
-static const size_t UNKNOWN_ERROR_BUFSIZE = 32;
-__thread char unknown_error_buf[UNKNOWN_ERROR_BUFSIZE];
+const size_t ERROR_BUFSIZE = 64;
+__thread char tls_error_buf[ERROR_BUFSIZE];
 
 int DescribeCustomizedErrno(
     int error_code, const char* error_name, const char* description) {
@@ -32,20 +32,18 @@ int DescribeCustomizedErrno(
               error_name, error_code);
     }
     const char* desc = errno_desc[error_code - ERRNO_BEGIN];
-    if (!desc) {
-        // g++ 4.8.2 reports nonnull warning for directly using NULL as the
-        // second parameter to strerror_r which is totally valid.
-        char* cheat_nonnull = NULL;
-        desc = strerror_r(error_code, cheat_nonnull, 0);
-    }
     if (desc) {
         if (strcmp(desc, description) == 0) {
             fprintf(stderr, "WARNING: Detected shared library loading\n");
-            return 0;
+            return -1;
+        }
+    } else {
+        desc = strerror_r(error_code, tls_error_buf, ERROR_BUFSIZE);
+        if (desc && strncmp(desc, "Unknown error", 13) != 0) {
+            error(EXIT_FAILURE, 0,
+                    "Fail to define %s(%d) which is already defined as `%s', abort.",
+                    error_name, error_code, desc);
         }
-        error(EXIT_FAILURE, 0,
-              "Fail to define %s(%d) which is already defined as `%s', abort.",
-              error_name, error_code, desc);
     }
     errno_desc[error_code - ERRNO_BEGIN] = description;
     return 0;  // must
@@ -55,24 +53,21 @@ int DescribeCustomizedErrno(
 
 const char* berror(int error_code) {
     if (error_code == -1) {
-        return "General Error(-1)";
+        return "General Error -1";
     }
     if (error_code >= base::ERRNO_BEGIN && error_code < base::ERRNO_END) {
         const char* s = base::errno_desc[error_code - base::ERRNO_BEGIN];
         if (s) {
             return s;
         }
-        // g++ 4.8.2 reports nonnull warning for directly using NULL as the
-        // second parameter to strerror_r which is totally valid.
-        char* cheat_nonnull = NULL;
-        s = strerror_r(error_code, cheat_nonnull, 0);
+        s = strerror_r(error_code, base::tls_error_buf, base::ERROR_BUFSIZE);
         if (s) {  // strerror_r returns NULL if error_code is unknown
             return s;
         }
     }
-    snprintf(base::unknown_error_buf, base::UNKNOWN_ERROR_BUFSIZE,
-             "Unknown Error(%d)", error_code);
-    return base::unknown_error_buf;
+    snprintf(base::tls_error_buf, base::ERROR_BUFSIZE,
+             "Unknown Error %d", error_code);
+    return base::tls_error_buf;
 }
 
 const char* berror() {

base/files/file_enumerator_posix.cc

@@ -129,9 +129,17 @@ bool FileEnumerator::ReadDirectory(std::vector<FileInfo>* entries,
          additional space for pathname may be needed
 #endif
 
-  struct dirent dent_buf;
   struct dirent* dent;
+  // readdir_r is marked as deprecated since glibc 2.24. 
+  // Using readdir on _different_ DIR* object is already thread-safe in
+  // most modern libc implementations.
+#if defined(__GLIBC__) &&  \
+    (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 24))
+  while ((dent = readdir(dir))) {
+#else
+  struct dirent dent_buf;
   while (readdir_r(dir, &dent_buf, &dent) == 0 && dent) {
+#endif
     FileInfo info;
     info.filename_ = FilePath(dent->d_name);
 

base/iobuf.h

@@ -652,15 +652,17 @@ private:
 }  // namespace base
 
 // Specialize std::swap for IOBuf
+#if __cplusplus < 201103L  // < C++11
+#include <algorithm>  // std::swap until C++11
+#else
+#include <utility>  // std::swap since C++11
+#endif  // __cplusplus < 201103L
 namespace std {
-template <class T> void swap ( T& a, T& b );
-
 template <>
 inline void swap(base::IOBuf& a, base::IOBuf& b) {
     return a.swap(b);
 }
-
-};
+} // namespace std
 
 #include "base/iobuf_inl.h"
 

base/safe_strerror_posix.cc

@@ -9,7 +9,11 @@
 #include <stdio.h>
 #include <string.h>
 
-#define USE_HISTORICAL_STRERRO_R (defined(__GLIBC__) || defined(OS_NACL))
+#if defined(__GLIBC__) || defined(OS_NACL)
+# define USE_HISTORICAL_STRERRO_R 1
+#else
+# define USE_HISTORICAL_STRERRO_R 0
+#endif
 
 #if USE_HISTORICAL_STRERRO_R && defined(__GNUC__)
 // GCC will complain about the unused second wrap function unless we tell it

base/strings/stringprintf.cc

@@ -10,6 +10,12 @@
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 
+// gcc7 reports that the first arg to vsnprintfT in StringAppendVT is NULL,
+// which I can't figure out why, turn off the warning right now.
+#if defined(__GNUC__) && __GNUC__ >= 7
+#pragma GCC diagnostic warning "-Wformat-truncation=0"
+#endif
+
 namespace base {
 
 namespace {

base/third_party/dmg_fp/dtoa.cc

@@ -1840,7 +1840,7 @@ gethex( CONST char **sp, U *rvp, int rounding, int sign)
 		switch(*++s) {
 		  case '-':
 			esign = 1;
-			/* no break */
+			// fall through
 		  case '+':
 			s++;
 		  }
@@ -2462,11 +2462,11 @@ strtod
 	for(s = s00;;s++) switch(*s) {
 		case '-':
 			sign = 1;
-			/* no break */
+			// fall through
 		case '+':
 			if (*++s)
 				goto break2;
-			/* no break */
+			// fall through
 		case 0:
 			goto ret0;
 		case '\t':
@@ -2570,6 +2570,7 @@ strtod
 		switch(c = *++s) {
 			case '-':
 				esign = 1;
+				// fall through
 			case '+':
 				c = *++s;
 			}
@@ -3755,7 +3756,7 @@ dtoa
 			break;
 		case 2:
 			leftright = 0;
-			/* no break */
+			// fall through
 		case 4:
 			if (ndigits <= 0)
 				ndigits = 1;
@@ -3763,7 +3764,7 @@ dtoa
 			break;
 		case 3:
 			leftright = 0;
-			/* no break */
+			// fall through
 		case 5:
 			i = ndigits + k + 1;
 			ilim = i;

base/third_party/icu/icu_utf.cc

@@ -159,10 +159,12 @@ utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c,
                 illegal=1;
                 break;
             }
+            // fall through
         case 2:
             trail=s[(i)++];
             (c)=((c)<<6)|(trail&0x3f);
             illegal|=(trail&0xc0)^0x80;
+            // fall through
         case 1:
             trail=s[(i)++];
             (c)=((c)<<6)|(trail&0x3f);

base/third_party/murmurhash3/murmurhash3.cpp

@@ -17,6 +17,11 @@
 #include <algorithm> // std::min
 #include "base/third_party/murmurhash3/murmurhash3.h"
 
+// Too many fallthroughs in this file to mark, just ignore the warning.
+#if defined(__GNUC__) && __GNUC__ >= 7
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+#endif
+
 //-----------------------------------------------------------------------------
 // Platform-specific functions and macros
 

brpc/amf.cpp

@@ -433,6 +433,7 @@ static bool ReadAMFObjectField(AMFInputStream* stream,
             LOG(ERROR) << "Fail to read class_name";
         }
     }
+    // fall through
     case AMF_MARKER_OBJECT: {
         if (field) {
             if (field->cpp_type() != google::protobuf::FieldDescriptor::CPPTYPE_MESSAGE) {
@@ -633,6 +634,7 @@ static bool ReadAMFObjectField(AMFInputStream* stream,
             LOG(ERROR) << "Fail to read class_name";
         }
     }
+    // fall through
     case AMF_MARKER_OBJECT: {
         if (!ReadAMFObjectBody(obj->MutableObject(name), stream)) {
             return false;
@@ -781,6 +783,7 @@ static bool ReadAMFArrayItem(AMFInputStream* stream, AMFArray* arr) {
             LOG(ERROR) << "Fail to read class_name";
         }
     }
+    // fall through
     case AMF_MARKER_OBJECT: {
         if (!ReadAMFObjectBody(arr->AddObject(), stream)) {
             return false;

brpc/builtin/dir_service.cpp

@@ -75,8 +75,14 @@ void DirService::default_method(::google::protobuf::RpcController* cntl_base,
 
         std::vector<std::string> files;
         files.reserve(32);
-        struct dirent ent;
-        for (struct dirent* p = &ent; readdir_r(dir, &ent, &p) == 0 && p; ) {
+        // readdir_r is marked as deprecated since glibc 2.24. 
+#if defined(__GLIBC__) && \
+        (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 24))
+        for (struct dirent* p = NULL; (p = readdir(dir)) != NULL; ) {
+#else
+        struct dirent entbuf;
+        for (struct dirent* p = NULL; readdir_r(dir, &entbuf, &p) == 0 && p; ) {
+#endif
             files.push_back(p->d_name);
         }
         CHECK_EQ(0, closedir(dir));

brpc/controller.cpp

@@ -664,6 +664,7 @@ void Controller::Call::OnComplete(Controller* c, int error_code/*note*/,
             }
             break;
         }
+        // fall through
     case CONNECTION_TYPE_SHORT:
         if (sending_sock != NULL) {
             // Check the comment in CONNECTION_TYPE_POOLED branch.

brpc/details/http_parser.cpp

@@ -2271,8 +2271,7 @@ http_parser_parse_url(const char *buf, size_t buflen, int is_connect,
 
       case s_req_server_with_at:
         found_at = 1;
-
-      /* FALLTROUGH */
+        // fall through
       case s_req_server:
         uf = UF_HOST;
         break;

brpc/policy/dh.cpp

@@ -22,30 +22,14 @@
 // CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 #include "base/logging.h"
+#include "base/ssl_compat.h"
 #include "brpc/policy/dh.h"
 
-
 namespace brpc {
 namespace policy {
 
-#define RFC2409_PRIME_1024                              \
-    "FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD1"  \
-    "29024E088A67CC74020BBEA63B139B22514A08798E3404DD"  \
-    "EF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245"  \
-    "E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7ED"  \
-    "EE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381"  \
-    "FFFFFFFFFFFFFFFF"
-
 void DHWrapper::clear() {
     if (_pdh != NULL) {
-        if (_pdh->p != NULL) {
-            BN_free(_pdh->p);
-            _pdh->p = NULL;
-        }
-        if (_pdh->g != NULL) {
-            BN_free(_pdh->g);
-            _pdh->g = NULL;
-        }
         DH_free(_pdh);
         _pdh = NULL;
     }
@@ -57,7 +41,9 @@ int DHWrapper::initialize(bool ensure_128bytes_public_key) {
             return -1;
         }
         if (ensure_128bytes_public_key) {
-            int key_size = BN_num_bytes(_pdh->pub_key);
+            const BIGNUM* pub_key = NULL;
+            DH_get0_key(_pdh, &pub_key, NULL);
+            int key_size = BN_num_bytes(pub_key);
             if (key_size != 128) {
                 RPC_VLOG << "regenerate 128B key, current=" << key_size;
                 clear();
@@ -70,15 +56,17 @@ int DHWrapper::initialize(bool ensure_128bytes_public_key) {
 }
     
 int DHWrapper::copy_public_key(char* pkey, int* pkey_size) const {
+    const BIGNUM* pub_key = NULL;
+    DH_get0_key(_pdh, &pub_key, NULL);
     // copy public key to bytes.
     // sometimes, the key_size is 127, seems ok.
-    int key_size = BN_num_bytes(_pdh->pub_key);
+    int key_size = BN_num_bytes(pub_key);
     CHECK_GT(key_size, 0);
         
     // maybe the key_size is 127, but dh will write all 128bytes pkey,
     // no need to set/initialize the pkey.
     // @see https://github.com/ossrs/srs/issues/165
-    key_size = BN_bn2bin(_pdh->pub_key, (unsigned char*)pkey);
+    key_size = BN_bn2bin(pub_key, (unsigned char*)pkey);
     CHECK_GT(key_size, 0);
         
     // output the size of public key.
@@ -107,40 +95,27 @@ int DHWrapper::copy_shared_key(const void* ppkey, int ppkey_size,
 }
     
 int DHWrapper::do_initialize() {
-    int bits_count = 1024; 
-        
-    //1. Create the DH
-    if ((_pdh = DH_new()) == NULL) {
-        LOG(ERROR) << "Fail to DH_new";
+    BIGNUM* p = get_rfc2409_prime_1024(NULL);
+    if (!p) {
         return -1;
     }
-    
-    //2. Create his internal p and g
-    if ((_pdh->p = BN_new()) == NULL) {
-        LOG(ERROR) << "Fail to BN_new _pdh->p";
+    // See RFC 2409, Section 6 "Oakley Groups"
+    // for the reason why 2 is used as generator.
+    BIGNUM* g = NULL;
+    BN_dec2bn(&g, "2");
+    if (!g) {
+        BN_free(p);
         return -1;
     }
-    if ((_pdh->g = BN_new()) == NULL) {
-        LOG(ERROR) << "Fail to BN_new _pdh->g";
+    _pdh = DH_new();
+    if (!_pdh) {
+        BN_free(p);
+        BN_free(g);
         return -1;
     }
+    DH_set0_pqg(_pdh, p, NULL, g);
     
-    //3. initialize p and g, @see ./test/ectest.c:260
-    if (!BN_hex2bn(&_pdh->p, RFC2409_PRIME_1024)) {
-        LOG(ERROR) << "Fail to BN_hex2bn _pdh->p";
-        return -1;
-    }
-    // @see ./test/bntest.c:1764
-    if (!BN_set_word(_pdh->g, 2)) {
-        LOG(ERROR) << "Fail to BN_set_word _pdh->g";
-        return -1;
-    }
-    
-    // 4. Set the key length
-    _pdh->length = bits_count;
-    
-    // 5. Generate private and public key
-    // @see ./test/dhtest.c:152
+    // Generate private and public key
     if (!DH_generate_key(_pdh)) {
         LOG(ERROR) << "Fail to DH_generate_key";
         return -1;
@@ -148,8 +123,6 @@ int DHWrapper::do_initialize() {
     return 0;
 }
 
-#undef RFC2409_PRIME_1024
-
 }  // namespace policy
 } // namespace brpc
 

brpc/policy/http_rpc_protocol.cpp

@@ -168,7 +168,7 @@ static void PrintMessage(const base::IOBuf& inbuf,
                          bool has_content) {
     base::IOBuf buf1 = inbuf;
     base::IOBuf buf2;
-    char str[32];
+    char str[48];
     if (request_or_response) {
         snprintf(str, sizeof(str), "[HTTP REQUEST @%s]", base::my_ip_cstr());
     } else {

brpc/redis_reply.cpp

@@ -255,6 +255,7 @@ void RedisReply::Print(std::ostream& os) const {
         break;
     case REDIS_REPLY_ERROR:
         os << "(error) ";
+        // fall through
     case REDIS_REPLY_STATUS:
         if (_length < sizeof(_data.short_str)) {
             os << _data.short_str;

brpc/serialized_request.cpp

@@ -120,7 +120,9 @@ const ::google::protobuf::Descriptor* SerializedRequest::descriptor() {
 }
 
 const SerializedRequest& SerializedRequest::default_instance() {
-    if (default_instance_ == NULL) protobuf_AddDesc_baidu_2frpc_2fserialized_5frequest_2eproto();  return *default_instance_;
+    if (default_instance_ == NULL)
+        protobuf_AddDesc_baidu_2frpc_2fserialized_5frequest_2eproto();
+    return *default_instance_;
 }
 
 SerializedRequest* SerializedRequest::default_instance_ = NULL;

bvar/collector.cpp

@@ -86,8 +86,7 @@ private:
     bool _stop;         // Set to true in dtor.
     pthread_t _grab_thread;     // For joining.
     pthread_t _dump_thread;
-
-    int64_t BAIDU_CACHELINE_ALIGNMENT _ngrab;
+    int64_t _ngrab BAIDU_CACHELINE_ALIGNMENT;
     int64_t _ndrop;
     int64_t _ndump;
     pthread_mutex_t _dump_thread_mutex;

bvar/default_variables.cpp

@@ -11,6 +11,7 @@
 #include "base/memory/singleton_on_pthread_once.h"
 #include "base/scoped_lock.h"
 #include "base/files/scoped_file.h"
+#include "base/files/file_enumerator.h"
 #include "base/file_util.h"
 #include "bvar/passive_status.h"
 
@@ -235,18 +236,15 @@ public:
 // ==================================================
 
 static int get_fd_count(int limit) {
-    DIR *dir = opendir("/proc/self/fd");
-    if (dir == NULL) {
-        PLOG_ONCE(WARNING) << "Fail to opendir /proc/self/fd";
-        return -1;
-    }
+    base::FileEnumerator fd_enum(base::FilePath("/proc/self/fd"),
+                                 false/*non recursive*/,
+                                 base::FileEnumerator::FILES);
     int count = 0;
-    dirent ent;
-    // We have to limit the scaning which consumes a lot of CPU when #fd
+    // Have to limit the scaning which consumes a lot of CPU when #fd
     // are huge (100k+)
-    for (dirent* p = &ent; readdir_r(dir, &ent, &p) == 0 && p &&
-             count <= limit; ++count) {}
-    closedir(dir);
+    for (base::FilePath name = fd_enum.Next();
+         !name.empty() && count <= limit;
+         name = fd_enum.Next(), ++count) {}
     return count - 2/*. and ..*/ - 1/*opendir itself*/;
 }
 

bvar/detail/agent_group.h

@@ -165,11 +165,10 @@ private:
         return *_s_free_ids;
     }
 
-    static pthread_mutex_t                                      _s_mutex;
-    static AgentId                                              _s_agent_kinds;
-    static std::deque<AgentId>                                  *_s_free_ids;
-    static __thread
-        BAIDU_CACHELINE_ALIGNMENT std::vector<ThreadBlock *>    *_s_tls_blocks;
+    static pthread_mutex_t                      _s_mutex;
+    static AgentId                              _s_agent_kinds;
+    static std::deque<AgentId>                  *_s_free_ids;
+    static __thread std::vector<ThreadBlock *>  *_s_tls_blocks;
 };
 
 template <typename Agent>

bvar/detail/combiner.h

@@ -7,8 +7,6 @@
 #define  BAIDU_BVAR__COMBINER_H
 
 #include <string>                       // std::string
-#include <algorithm>                    // std::swap until C++11
-#include <utility>                      // std::swap since C++11
 #include <vector>                       // std::vector
 #include "base/atomicops.h"             // base::atomic
 #include "base/scoped_lock.h"           // BAIDU_SCOPED_LOCK

test/Makefile

@@ -4,8 +4,11 @@ NEED_GMOCK=1
 include ../config.mk
 CPPFLAGS=-DBTHREAD_USE_FAST_PTHREAD_MUTEX -D__const__= -D_GNU_SOURCE -DUSE_SYMBOLIZE -DNO_TCMALLOC -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS
 CPPFLAGS+=-DUNIT_TEST -Dprivate=public -Dprotected=public -DBVAR_NOT_LINK_DEFAULT_VARIABLES
-CXXFLAGS=$(CPPFLAGS) -g -pipe -Wall -W -Werror -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x
-CFLAGS=$(CPPFLAGS) -g -pipe -Wall -W -Werror -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer
+CXXFLAGS=$(CPPFLAGS) -g -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-invalid-offsetof -Wno-unused-parameter -fno-omit-frame-pointer -std=c++0x
+CFLAGS=$(CPPFLAGS) -g -pipe -Wall -W -fPIC -fstrict-aliasing -Wno-unused-parameter -fno-omit-frame-pointer
+ifeq ($(shell test $(shell $(CXX) -dumpversion) -ge 7; echo $$?),0)
+CXXFLAGS+=-Wno-aligned-new
+endif
 
 HDRPATHS=-I. -I.. $(addprefix -I, $(HDRS))
 LIBPATHS=$(addprefix -L, $(LIBS))