Merge pull request #1292 from apache/http_improve

Compatibility improvement of protobuf json format and spring http spec

src/brpc/policy/http_rpc_protocol.cpp

@@ -21,6 +21,7 @@
 #include <json2pb/pb_to_json.h>                    // ProtoMessageToJson
 #include <json2pb/json_to_pb.h>                    // JsonToProtoMessage
 
+#include "brpc/policy/http_rpc_protocol.h"
 #include "butil/unique_ptr.h"                       // std::unique_ptr
 #include "butil/string_splitter.h"                  // StringMultiSplitter
 #include "butil/string_printf.h"
@@ -110,6 +111,7 @@ CommonStrings::CommonStrings()
     , CONTENT_TYPE_TEXT("text/plain")
     , CONTENT_TYPE_JSON("application/json")
     , CONTENT_TYPE_PROTO("application/proto")
+    , CONTENT_TYPE_SPRING_PROTO("application/x-protobuf")
     , ERROR_CODE("x-bd-error-code")
     , AUTHORIZATION("authorization")
     , ACCEPT_ENCODING("accept-encoding")
@@ -189,6 +191,9 @@ HttpContentType ParseContentType(butil::StringPiece ct, bool* is_grpc_ct) {
     } else if (ct.starts_with("proto")) {
         type = HTTP_CONTENT_PROTO;
         ct.remove_prefix(5);
+    } else if (ct.starts_with("x-protobuf")) {
+        type = HTTP_CONTENT_PROTO;
+        ct.remove_prefix(10);
     } else {
         return HTTP_CONTENT_OTHERS;
     }
@@ -511,7 +516,7 @@ void SerializeHttpRequest(butil::IOBuf* /*not used*/,
             opt.bytes_to_base64 = cntl->has_pb_bytes_to_base64();
             opt.jsonify_empty_array = cntl->has_pb_jsonify_empty_array();
             opt.always_print_primitive_fields = cntl->has_always_print_primitive_fields();
-            
+
             opt.enum_option = (FLAGS_pb_enum_as_number
                                ? json2pb::OUTPUT_ENUM_BY_NUMBER
                                : json2pb::OUTPUT_ENUM_BY_NAME);

src/brpc/policy/http_rpc_protocol.h

@@ -37,6 +37,7 @@ struct CommonStrings {
     std::string CONTENT_TYPE_TEXT;
     std::string CONTENT_TYPE_JSON;
     std::string CONTENT_TYPE_PROTO;
+    std::string CONTENT_TYPE_SPRING_PROTO;
     std::string ERROR_CODE;
     std::string AUTHORIZATION;
     std::string ACCEPT_ENCODING;

src/json2pb/json_to_pb.cpp

@@ -24,6 +24,7 @@
 #include <typeinfo>
 #include <limits> 
 #include <google/protobuf/descriptor.h>
+#include "butil/strings/string_number_conversions.h"
 #include "json_to_pb.h"
 #include "zero_copy_stream_reader.h"       // ZeroCopyStreamReader
 #include "encode_decode.h"
@@ -207,10 +208,63 @@ inline bool convert_enum_type(const BUTIL_RAPIDJSON_NAMESPACE::Value&item, bool
     return true;
 }
 
+inline bool convert_int64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item, bool repeated,
+                               google::protobuf::Message* message,
+                               const google::protobuf::FieldDescriptor* field, 
+                               const google::protobuf::Reflection* reflection,
+                               std::string* err) { 
+  
+    int64_t num;
+    if (item.IsInt64()) {
+        if (repeated) {
+            reflection->AddInt64(message, field, item.GetInt64());
+        } else {
+            reflection->SetInt64(message, field, item.GetInt64());
+        }
+    } else if (item.IsString() &&
+               butil::StringToInt64({item.GetString(), item.GetStringLength()},
+                                    &num)) {
+        if (repeated) {
+            reflection->AddInt64(message, field, num);
+        } else {
+            reflection->SetInt64(message, field, num);
+        }
+    } else {
+        return value_invalid(field, "INT64", item, err);
+    }
+    return true;
+}
+
+inline bool convert_uint64_type(const BUTIL_RAPIDJSON_NAMESPACE::Value& item,
+                                bool repeated,
+                                google::protobuf::Message* message,
+                                const google::protobuf::FieldDescriptor* field,
+                                const google::protobuf::Reflection* reflection,
+                                std::string* err) {
+    uint64_t num;
+    if (item.IsUint64()) {
+        if (repeated) {
+            reflection->AddUInt64(message, field, item.GetUint64());
+        } else {
+            reflection->SetUInt64(message, field, item.GetUint64());
+        }
+    } else if (item.IsString() &&
+               butil::StringToUint64({item.GetString(), item.GetStringLength()},
+                                     &num)) {
+        if (repeated) {
+            reflection->AddUInt64(message, field, num);
+        } else {
+            reflection->SetUInt64(message, field, num);
+        }
+    } else {
+        return value_invalid(field, "UINT64", item, err);
+    }
+    return true;
+}
+
 bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
                              google::protobuf::Message* message,
-                             const Json2PbOptions& options,
-                             std::string* err);
+                             const Json2PbOptions& options, std::string* err);
 
 //Json value to protobuf convert rules for type:
 //Json value type                 Protobuf type                convert rules
@@ -219,9 +273,10 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
 //int64                           int uint int64 uint64        valid convert is available
 //uint64                          int uint int64 uint64        valid convert is available
 //int uint int64 uint64           float double                 available
-//"NaN" "Infinity" "-Infinity"    float double                 only "NaN" "Infinity" "-Infinity" is available    
+//"NaN" "Infinity" "-Infinity"    float double                 only "NaN" "Infinity" "-Infinity" is available
 //int                             enum                         valid enum number value is available
-//string                          enum                         valid enum name value is available         
+//string                          enum                         valid enum name value is available
+//string                          int64 uint64                 valid convert is available
 //other mismatch type convertion will be regarded as error.
 #define J2PCHECKTYPE(value, cpptype, jsontype) ({                   \
             MatchType match_type = TYPE_MATCH;                      \
@@ -234,6 +289,7 @@ bool JsonValueToProtoMessage(const BUTIL_RAPIDJSON_NAMESPACE::Value& json_value,
             match_type;                                             \
         })
 
+
 static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
                                   const google::protobuf::FieldDescriptor* field,
                                   google::protobuf::Message* message,
@@ -271,15 +327,48 @@ static bool JsonValueToProtoField(const BUTIL_RAPIDJSON_NAMESPACE::Value& value,
                 reflection->Set##method(message, field, value.Get##jsontype()); \
             }                                                           \
             break;                                                      \
-        }                                                           
+        }                                                               \
+          
         CASE_FIELD_TYPE(INT32,  Int32,  Int);
         CASE_FIELD_TYPE(UINT32, UInt32, Uint);
         CASE_FIELD_TYPE(BOOL,   Bool,   Bool);
-        CASE_FIELD_TYPE(INT64,  Int64,  Int64);
-        CASE_FIELD_TYPE(UINT64, UInt64, Uint64);
 #undef CASE_FIELD_TYPE
 
-    case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:  
+    case google::protobuf::FieldDescriptor::CPPTYPE_INT64:
+        if (field->is_repeated()) {
+            const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
+            for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size;
+                 ++index) {
+                const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index];
+                if (!convert_int64_type(item, true, message, field, reflection,
+                                        err)) {
+                    return false;
+                }
+            }
+        } else if (!convert_int64_type(value, false, message, field, reflection,
+                                       err)) {
+            return false;
+        }
+        break;
+
+    case google::protobuf::FieldDescriptor::CPPTYPE_UINT64:
+        if (field->is_repeated()) {
+            const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
+            for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size;
+                 ++index) {
+                const BUTIL_RAPIDJSON_NAMESPACE::Value& item = value[index];
+                if (!convert_uint64_type(item, true, message, field, reflection,
+                                         err)) {
+                    return false;
+                }
+            }
+        } else if (!convert_uint64_type(value, false, message, field, reflection,
+                                       err)) {
+            return false;
+        }
+        break;
+
+    case google::protobuf::FieldDescriptor::CPPTYPE_FLOAT:
         if (field->is_repeated()) {
             const BUTIL_RAPIDJSON_NAMESPACE::SizeType size = value.Size();
             for (BUTIL_RAPIDJSON_NAMESPACE::SizeType index = 0; index < size; ++index) {

src/json2pb/pb_to_json.cpp

@@ -51,7 +51,7 @@ public:
     bool Convert(const google::protobuf::Message& message, Handler& handler);
 
     const std::string& ErrorText() const { return _error; }
-    
+
 private:
     template <typename Handler>
     bool _PbFieldToJson(const google::protobuf::Message& message,

test/brpc_http_rpc_protocol_unittest.cpp

@@ -1443,4 +1443,41 @@ TEST_F(HttpTest, http2_handle_goaway_streams) {
         brpc::Join(ids[i]);
     }
 }
+
+TEST_F(HttpTest, spring_protobuf_content_type) {
+    const int port = 8923;
+    brpc::Server server;
+    EXPECT_EQ(0, server.AddService(&_svc, brpc::SERVER_DOESNT_OWN_SERVICE));
+    EXPECT_EQ(0, server.Start(port, nullptr));
+
+    brpc::Channel channel;
+    brpc::ChannelOptions options;
+    options.protocol = "http";
+    ASSERT_EQ(0, channel.Init(butil::EndPoint(butil::my_ip(), port), &options));
+
+    brpc::Controller cntl;
+    test::EchoRequest req;
+    test::EchoResponse res;
+    req.set_message(EXP_REQUEST);
+    cntl.http_request().set_method(brpc::HTTP_METHOD_POST);
+    cntl.http_request().uri() = "/EchoService/Echo";
+    cntl.http_request().set_content_type("application/x-protobuf");
+    cntl.request_attachment().append(req.SerializeAsString());
+    channel.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr);
+    ASSERT_FALSE(cntl.Failed());
+    ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type());
+    ASSERT_TRUE(res.ParseFromString(cntl.response_attachment().to_string()));
+    ASSERT_EQ(EXP_RESPONSE, res.message());
+
+    brpc::Controller cntl2;
+    test::EchoService_Stub stub(&channel);
+    req.set_message(EXP_REQUEST);
+    res.Clear();
+    cntl2.http_request().set_content_type("application/x-protobuf");
+    stub.Echo(&cntl2, &req, &res, nullptr);
+    ASSERT_FALSE(cntl.Failed());
+    ASSERT_EQ(EXP_RESPONSE, res.message());
+    ASSERT_EQ("application/x-protobuf", cntl.http_response().content_type());
+}
+
 } //namespace

test/brpc_protobuf_json_unittest.cpp

@@ -1308,7 +1308,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_encode_decode_perf_case) {
 }
 
 TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
-    
+
     std::ifstream in("jsonout", std::ios::in);
     std::ostringstream tmp;
     tmp << in.rdbuf();
@@ -1317,8 +1317,8 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
 
     printf("----------test pb to json performance------------\n\n");
 
-    std::string error; 
-  
+    std::string error;
+
     butil::Timer timer;
     bool res;
     float avg_time1 = 0;
@@ -1331,7 +1331,7 @@ TEST_F(ProtobufJsonTest, pb_to_json_complex_perf_case) {
     res = JsonToProtoMessage(info3, &data, option, &error);
     timer.stop();
     avg_time1 += timer.u_elapsed();
-    ASSERT_TRUE(res);
+    ASSERT_TRUE(res) << error;
     ProfilerStart("pb_to_json_complex_perf.prof");
     for (int i = 0; i < times; i++) { 
         std::string error1;
@@ -1460,4 +1460,15 @@ TEST_F(ProtobufJsonTest, extension_case) {
     ASSERT_EQ("{\"hobby\":\"coding\",\"name\":\"hello\",\"id\":9,\"datadouble\":2.2,\"datafloat\":1.0}", output);
 }
 
+TEST_F(ProtobufJsonTest, string_to_int64) {
+    auto json = R"({"name":"hello", "id":9, "data": "123456", "datadouble":2.2, "datafloat":1.0})";
+    Person person;
+    std::string err;
+    ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person, &err)) << err;
+    ASSERT_EQ(person.data(), 123456);
+    json = R"({"name":"hello", "id":9, "data": 1234567, "datadouble":2.2, "datafloat":1.0})";
+    ASSERT_TRUE(json2pb::JsonToProtoMessage(json, &person));
+    ASSERT_EQ(person.data(), 1234567);
+}
+
 }