Merge branch 'master' of ssh://icode.baidu.com:8235/baidu/opensource/baidu-rpc

src/json2pb/json_to_pb.cpp

@@ -10,6 +10,7 @@
 #include "json_to_pb.h"
 #include "zero_copy_stream_reader.h"       // ZeroCopyStreamReader
 #include "encode_decode.h"
+#include "base/base64.h"
 #include "base/string_printf.h"
 #include "protobuf_map.h"
 #include "rapidjson.h"
@@ -24,6 +25,14 @@
 
 namespace json2pb {
 
+Json2PbOptions::Json2PbOptions()
+#ifdef BAIDU_INTERNAL
+    : base64_to_bytes(false) {
+#else
+    : base64_to_bytes(true) {
+#endif
+}
+
 enum MatchType { 
     TYPE_MATCH = 0x00, 
     REQUIRED_OR_REPEATED_TYPE_MISMATCH = 0x01, 
@@ -182,7 +191,9 @@ inline bool convert_enum_type(const rapidjson::Value&item, bool repeated,
 }
 
 bool JsonValueToProtoMessage(const rapidjson::Value& json_value,
-                             google::protobuf::Message* message, std::string* err);
+                             google::protobuf::Message* message,
+                             const Json2PbOptions& options,
+                             std::string* err);
 
 //Json value to protobuf convert rules for type:
 //Json value type                 Protobuf type                convert rules
@@ -208,7 +219,9 @@ bool JsonValueToProtoMessage(const rapidjson::Value& json_value,
 
 static bool JsonValueToProtoField(const rapidjson::Value& value,
                                   const google::protobuf::FieldDescriptor* field,
-                                  google::protobuf::Message* message, std::string* err) {
+                                  google::protobuf::Message* message,
+                                  const Json2PbOptions& options,
+                                  std::string* err) {
     if (value.IsNull()) {
         if (field->is_required()) {
             J2PERROR(err, "Missing required field: %s", field->full_name().c_str());
@@ -288,11 +301,29 @@ static bool JsonValueToProtoField(const rapidjson::Value& value,
                 const rapidjson::Value & item = value[index];
                 if (TYPE_MATCH == J2PCHECKTYPE(item, string, String)) { 
                     std::string str(item.GetString(), item.GetStringLength());
+                    if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES &&
+                        options.base64_to_bytes) {
+                        std::string str_decoded;
+                        if (!base::Base64Decode(str, &str_decoded)) {
+                            J2PERROR(err, "Fail to decode base64 string=%s", str.c_str());
+                            return false;
+                        }
+                        str = str_decoded;
+                    }
                     reflection->AddString(message, field, str);
                 }  
             }
         } else if (TYPE_MATCH == J2PCHECKTYPE(value, string, String)) {
             std::string str(value.GetString(), value.GetStringLength());
+            if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES &&
+                options.base64_to_bytes) {
+                std::string str_decoded;
+                if (!base::Base64Decode(str, &str_decoded)) {
+                    J2PERROR(err, "Fail to decode base64 string=%s", str.c_str());
+                    return false;
+                }
+                str = str_decoded;
+            }
             reflection->SetString(message, field, str);
         }
         break;
@@ -320,13 +351,13 @@ static bool JsonValueToProtoField(const rapidjson::Value& value,
                 const rapidjson::Value& item = value[index];
                 if (TYPE_MATCH == J2PCHECKTYPE(item, message, Object)) { 
                     if (!JsonValueToProtoMessage(
-                            item, reflection->AddMessage(message, field), err)) {
+                            item, reflection->AddMessage(message, field), options, err)) {
                         return false;
                     }
                 } 
             }
         } else if (!JsonValueToProtoMessage(
-            value, reflection->MutableMessage(message, field), err)) {
+            value, reflection->MutableMessage(message, field), options, err)) {
             return false;
         }
         break;
@@ -336,7 +367,9 @@ static bool JsonValueToProtoField(const rapidjson::Value& value,
 
 bool JsonMapToProtoMap(const rapidjson::Value& value,
                        const google::protobuf::FieldDescriptor* map_desc,
-                       google::protobuf::Message* message, std::string* err) {
+                       google::protobuf::Message* message,
+                       const Json2PbOptions& options,
+                       std::string* err) {
     if (!value.IsObject()) {
         J2PERROR(err, "Non-object value for map field: %s",
                  map_desc->full_name().c_str());
@@ -356,7 +389,7 @@ bool JsonMapToProtoMap(const rapidjson::Value& value,
         entry_reflection->SetString(
             entry, key_desc, std::string(it->name.GetString(),
                                          it->name.GetStringLength()));
-        if (!JsonValueToProtoField(it->value, value_desc, entry, err)) {
+        if (!JsonValueToProtoField(it->value, value_desc, entry, options, err)) {
             return false;
         }
     }
@@ -364,7 +397,9 @@ bool JsonMapToProtoMap(const rapidjson::Value& value,
 }
 
 bool JsonValueToProtoMessage(const rapidjson::Value& json_value,
-                             google::protobuf::Message* message, std::string* err) {
+                             google::protobuf::Message* message,
+                             const Json2PbOptions& options,
+                             std::string* err) {
     if (!json_value.IsObject()) {
         J2PERROR(err, "`json_value' is not a json object");
         return false;
@@ -426,11 +461,11 @@ bool JsonValueToProtoMessage(const rapidjson::Value& json_value,
 
         if (IsProtobufMap(field) && value_ptr->IsObject()) {
             // Try to parse json like {"key":value, ...} into protobuf map
-            if (!JsonMapToProtoMap(*value_ptr, field, message, err)) {
+            if (!JsonMapToProtoMap(*value_ptr, field, message, options, err)) {
                 return false;
             }
         } else {
-            if (!JsonValueToProtoField(*value_ptr, field, message, err)) {
+            if (!JsonValueToProtoField(*value_ptr, field, message, options, err)) {
                 return false;
             }
         }
@@ -447,19 +482,42 @@ bool ZeroCopyStreamToJson(rapidjson::Document *dest,
 
 inline bool JsonToProtoMessageInline(const std::string& json_string, 
                         google::protobuf::Message* message,
+                        const Json2PbOptions& options,
                         std::string* error) {
     if (error) {
         error->clear();
     }
     rapidjson::Document d;
     d.Parse<0>(json_string.c_str());
-    return json2pb::JsonValueToProtoMessage(d, message, error);
+    return json2pb::JsonValueToProtoMessage(d, message, options, error);
+}
+
+bool JsonToProtoMessage(const std::string& json_string,
+                        google::protobuf::Message* message,
+                        const Json2PbOptions& options,
+                        std::string* error) {
+    return JsonToProtoMessageInline(json_string, message, options, error);
+}
+
+bool JsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* stream,
+                        google::protobuf::Message* message,
+                        const Json2PbOptions& options,
+                        std::string* error) {
+    if (error) {
+        error->clear();
+    }
+    rapidjson::Document d;
+    if (!json2pb::ZeroCopyStreamToJson(&d, stream)) {
+        J2PERROR(error, "Invalid json format");
+        return false;
+    }
+    return json2pb::JsonValueToProtoMessage(d, message, options, error);
 }
 
 bool JsonToProtoMessage(const std::string& json_string, 
                         google::protobuf::Message* message,
                         std::string* error) {
-    return JsonToProtoMessageInline(json_string, message, error); 
+    return JsonToProtoMessageInline(json_string, message, Json2PbOptions(), error);
 }
 
 // For ABI compatibility with 1.0.0.0
@@ -469,7 +527,7 @@ bool JsonToProtoMessage(const std::string& json_string,
 bool JsonToProtoMessage(std::string json_string, 
                         google::protobuf::Message* message,
                         std::string* error) {
-    return JsonToProtoMessageInline(json_string, message, error); 
+    return JsonToProtoMessageInline(json_string, message, Json2PbOptions(), error);
 }
 
 bool JsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream *stream,
@@ -483,7 +541,7 @@ bool JsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream *stream,
         J2PERROR(error, "Invalid json format");
         return false;
     }
-    return json2pb::JsonValueToProtoMessage(d, message, error);
+    return json2pb::JsonValueToProtoMessage(d, message, Json2PbOptions(), error);
 }
 } //namespace json2pb
 

src/json2pb/json_to_pb.h

@@ -11,6 +11,16 @@
 #include <google/protobuf/io/zero_copy_stream.h>    // ZeroCopyInputStream
 
 namespace json2pb {
+
+struct Json2PbOptions {
+    Json2PbOptions();
+
+    // Decode string in json using base64 decoding if the type of
+    // corresponding field is bytes when this option is turned on.
+    // Default: false for baidu-interal, true otherwise.
+    bool base64_to_bytes;
+};
+
 // Rules: http://wiki.baidu.com/display/RPC/Json+%3C%3D%3E+Protobuf
 
 // Convert `json' to protobuf `message'.
@@ -18,12 +28,23 @@ namespace json2pb {
 // message on failure.
 bool JsonToProtoMessage(const std::string& json,
                         google::protobuf::Message* message,
+                        const Json2PbOptions& options,
                         std::string* error = NULL);
 
 // send output to ZeroCopyOutputStream instead of std::string.
 bool JsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream *json,
+                        google::protobuf::Message* message,
+                        const Json2PbOptions& options,
+                        std::string* error = NULL);
+
+// Using default Json2PbOptions.
+bool JsonToProtoMessage(const std::string& json,
+                        google::protobuf::Message* message,
+                        std::string* error = NULL);
+
+bool JsonToProtoMessage(google::protobuf::io::ZeroCopyInputStream* stream,
                         google::protobuf::Message* message,
                         std::string* error = NULL);
-}
+} // namespace json2pb
 
 #endif // BRPC_JSON2PB_JSON_TO_PB_H

src/json2pb/pb_to_json.cpp

@@ -5,6 +5,7 @@
 #include <sys/time.h>
 #include <time.h>
 #include <google/protobuf/descriptor.h>
+#include "base/base64.h"
 #include "zero_copy_stream_writer.h"
 #include "encode_decode.h"
 #include "protobuf_map.h"
@@ -15,7 +16,12 @@ namespace json2pb {
 Pb2JsonOptions::Pb2JsonOptions()
     : enum_option(OUTPUT_ENUM_BY_NAME)
     , pretty_json(false)
-    , enable_protobuf_map(true) {
+    , enable_protobuf_map(true)
+#ifdef BAIDU_INTERNAL
+    , bytes_to_base64(false) {
+#else
+    , bytes_to_base64(true) {
+#endif
 }
 
 class PbToJsonConverter {
@@ -175,13 +181,27 @@ bool PbToJsonConverter::_PbFieldToJson(
             for (int index = 0; index < field_size; ++index) {
                 value = reflection->GetRepeatedStringReference(
                     message, field, index, &value);
-                handler.String(value.data(), value.size(), false);
+                if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES
+                    && _option.bytes_to_base64) {
+                    std::string value_decoded;
+                    base::Base64Encode(value, &value_decoded);
+                    handler.String(value_decoded.data(), value_decoded.size(), false);
+                } else {
+                    handler.String(value.data(), value.size(), false);
+                }
             }
             handler.EndArray(field_size);
             
         } else {
             value = reflection->GetStringReference(message, field, &value);
-            handler.String(value.data(), value.size(), false);
+            if (field->type() == google::protobuf::FieldDescriptor::TYPE_BYTES
+                && _option.bytes_to_base64) {
+                std::string value_decoded;
+                base::Base64Encode(value, &value_decoded);
+                handler.String(value_decoded.data(), value_decoded.size(), false);
+            } else {
+                handler.String(value.data(), value.size(), false);
+            }
         }
         break;
     }

src/json2pb/pb_to_json.h

@@ -28,12 +28,18 @@ struct Pb2JsonOptions {
     // Use rapidjson::PrettyWriter to generate the json when this option is on.
     // NOTE: currently PrettyWriter is not optimized yet thus the conversion
     // functions may be slower when this option is turned on.
+    // Default: false
     bool pretty_json;
 
-    // true, we will automatically convert
-    // "repeated { required string key = 1; required string value = 2; }"
-    // to a map object of json and vice versa
+    // Convert "repeated { required string key = 1; required string value = 2; }"
+    // to a map object of json and vice versa when this option is turned on.
+    // Default: true
     bool enable_protobuf_map;
+
+    // Encode the field of type bytes to string in json using base64
+    // encoding when this option is turned on.
+    // Default: false for baidu-internal, true otherwise.
+    bool bytes_to_base64;
 };
 
 // Rules: http://wiki.baidu.com/display/RPC/Json+%3C%3D%3E+Protobuf

test/Makefile

@@ -133,6 +133,7 @@ TEST_BASE_SOURCES = \
     test_file_util_linux.cc \
     base_unittest_main.cpp
 
+
 TEST_BASE_OBJS = $(addsuffix .o, $(basename $(TEST_BASE_SOURCES))) 
 
 TEST_BVAR_SOURCES = $(wildcard bvar_*_unittest.cpp)

test/addressbook.proto

+package addressbook;
+message Person {
+    required string name = 1;
+    required int32 id = 2;        // Unique ID number for this person.
+    optional string email = 3;
+
+    enum PhoneType {
+        MOBILE = 0;
+        HOME = 1;
+        WORK = 2;
+    }
+
+    message PhoneNumber {
+        required string number = 1;
+        optional PhoneType type = 2 [default = HOME];
+    }
+
+    repeated PhoneNumber phone = 4;
+
+    optional int64 data = 5;
+  
+    optional sint32 data32 = 6;
+  
+    optional sint64 data64 = 7;
+
+    required double datadouble = 8;
+
+    required float  datafloat = 9; 
+  
+    optional uint32 datau32 = 10;
+  
+    optional uint64 datau64 = 11;
+
+    optional bool   databool = 12;
+
+    optional bytes databyte = 13;
+  
+    optional fixed32 datafix32 = 14;
+  
+    optional fixed64 datafix64 = 15;
+  
+    optional sfixed32 datasfix32 = 16;
+  
+    optional sfixed64 datasfix64 = 17;
+
+    optional float datafloat_scientific = 18;
+    
+    optional double datadouble_scientific = 19;
+
+    extensions 100 to 200;
+}
+
+extend Person {
+    optional string hobby = 100;
+}
+
+message AddressBook {
+    repeated Person person = 1;
+}
+

test/addressbook1.proto

+message Ext {
+  optional fixed32 age = 2;
+  required bytes databyte = 3;
+  enum PhoneType {
+    MOBILE = 0;
+    HOME = 1;
+    WORK = 2;
+  }
+
+  optional PhoneType enumtype = 4 [default = HOME];
+} 
+
+message Content {
+  optional string uid = 1;
+  required float distance = 16;
+  optional Ext ext = 17;
+}
+
+message JsonContextBody {
+  optional int64 type = 1;
+  repeated int32 data = 5;
+  repeated string info = 6;
+  required bool  judge = 2;
+  required double spur = 3;
+  repeated Content content = 4;
+  optional float text = 7;
+}
+
+
+message PersonInfo {
+  optional string name = 1;
+  optional int32 id = 2;        // Unique ID number for this person.
+  optional JsonContextBody json_body = 18;
+}
+
+message AddressBook {
+  repeated PersonInfo person = 1;
+}
+

test/addressbook_encode_decode.proto

+message ExtEncDec {
+  optional fixed32 Aa_ge_Z040_ = 2;
+  optional bytes databyte_Z040_std_Z058__Z058_string_Z041_ = 3;
+  enum PhoneTypeEncDec {
+    MOBILE = 0;
+    HOME = 1;
+    WORK = 2;
+  }
+
+  optional PhoneTypeEncDec enum_Z045__Z045_type = 4 [default = HOME];
+} 
+
+message ContentEncDec {
+  optional string uid_Z042_ = 1;
+  required float Distance_info_ = 16;
+  optional ExtEncDec _ext_Z037_T_ = 17;
+}
+
+message JsonContextBodyEncDec {
+  repeated string info = 6;
+  optional int64 type = 1;
+  repeated int32 data_Z058_array = 5;
+  required bool  judge = 2;
+  optional double spur = 3;
+  repeated ContentEncDec _Z064_Content_Test_Z037__Z064_ = 4;
+}
+
+
+message PersonEncDec {
+  optional string name = 1;
+  optional int32 id = 2;        // Unique ID number for this person.
+  optional JsonContextBodyEncDec json_body = 18;
+}
+
+message AddressBookEncDec {
+  repeated PersonEncDec person = 1;
+}
+

test/addressbook_map.proto

+message AddressNoMap {
+    required string addr = 1;
+}
+
+message AddressIntMap {
+    required string addr = 1;
+    message MapFieldEntry {
+        required string key = 1;
+        required int32 value = 2;
+    }
+    repeated MapFieldEntry numbers = 2;
+}
+
+message AddressStringMap {
+    required string addr = 1;
+    message MapFieldEntry {
+        required string key = 1;
+        required string value = 2;
+    }
+    repeated MapFieldEntry contacts = 2;
+}
+
+message AddressComplex {
+    required string addr = 1;
+    
+    message FriendEntry {
+        required string key = 1;
+        message Education {
+            required string school = 1;
+            required int32 year = 2;
+        }
+        repeated Education value = 2;
+    }
+    repeated FriendEntry friends = 2;
+}
+
+message haha {
+    repeated int32 a = 1;
+}
\ No newline at end of file

test/brpc_repeated_field_unittest.cpp

+// Copyright (c) 2015 Baidu.com, Inc. All Rights Reserved
+
+// Author: Zhangyi Chen (chenzhangyi01@baidu.com)
+// Date: 2015/10/10 17:55:13
+
+#include "repeated.pb.h"
+#include <gtest/gtest.h>
+#include <json2pb/pb_to_json.h>
+
+class RepeatedFieldTest : public testing::Test {
+protected:
+    void SetUp() {}
+    void TearDown() {}
+};
+
+TEST_F(RepeatedFieldTest, empty_array) {
+    RepeatedMessage m;
+    std::string json;
+
+    ASSERT_TRUE(json2pb::ProtoMessageToJson(m, &json));
+    std::cout << json << std::endl;
+
+    m.add_strings();
+    m.add_ints(1);
+    m.add_msgs();
+    json.clear();
+    ASSERT_TRUE(json2pb::ProtoMessageToJson(m, &json));
+    std::cout << json << std::endl;
+}

test/repeated.proto

+
+message Dummy {}
+
+message RepeatedMessage {
+    repeated int32 ints = 1;
+    repeated string strings = 2;
+    repeated Dummy msgs = 3;
+}

tools/patch_from_svn

@@ -43,6 +43,10 @@ cat $PATCHFILE | sed -e 's/src\/baidu\/rpc\//src\/brpc\//g' \
             -e 's/<\(base\/[^>]*\)>/"\1"/g' \
             -e 's/<\(bthread\/[^>]*\)>/"\1"/g' \
             -e 's/<\(mcpack2pb\/[^>]*\)>/"\1"/g' \
+            -e 's/\<protobuf_json\>/json2pb/g' \
+            -e 's/src\/json_to_pb/src\/json2pb\/json_to_pb/g' \
+            -e 's/src\/pb_to_json/src\/json2pb\/pb_to_json/g' \
+            -e 's/test\/test_protobuf_json/test\/brpc_protobuf_json_unittest/g' \
             -e 's/ base\// src\/base\//g' \
             -e 's/ bthread\// src\/bthread\//g' \
             -e 's/ bvar\// src\/bvar\//g' \