Unverified Commit cecba296 authored by Paul Yang's avatar Paul Yang Committed by GitHub

Down-integrate internal changes (#5467)

* Down-integrate internal changes

* Update generated code for php, objc and csharp.

* Add missing dependency in conformance_php
parent f93cf5a3
......@@ -21,7 +21,9 @@ add_custom_command(
add_executable(conformance_test_runner
${protobuf_source_dir}/conformance/conformance.pb.cc
${protobuf_source_dir}/conformance/conformance_test.cc
${protobuf_source_dir}/conformance/conformance_test_impl.cc
${protobuf_source_dir}/conformance/binary_json_conformance_main.cc
${protobuf_source_dir}/conformance/binary_json_conformance_suite.cc
${protobuf_source_dir}/conformance/binary_json_conformance_suite.h
${protobuf_source_dir}/conformance/conformance_test_runner.cc
${protobuf_source_dir}/conformance/third_party/jsoncpp/json.h
${protobuf_source_dir}/conformance/third_party/jsoncpp/jsoncpp.cpp
......
......@@ -5,6 +5,7 @@ set(libprotobuf_lite_files
${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc
${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc
${protobuf_source_dir}/src/google/protobuf/parse_context.cc
${protobuf_source_dir}/src/google/protobuf/io/coded_stream.cc
${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc
${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
......@@ -30,6 +31,7 @@ set(libprotobuf_lite_includes
${protobuf_source_dir}/src/google/protobuf/extension_set.h
${protobuf_source_dir}/src/google/protobuf/generated_message_util.h
${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.h
${protobuf_source_dir}/src/google/protobuf/parse_context.h
${protobuf_source_dir}/src/google/protobuf/io/coded_stream.h
${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.h
${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.h
......
......@@ -206,7 +206,9 @@ EXTRA_DIST = \
conformance_test_runner_LDADD = $(top_srcdir)/src/libprotobuf.la
conformance_test_runner_SOURCES = conformance_test.h conformance_test.cc \
conformance_test_impl.cc \
binary_json_conformance_main.cc \
binary_json_conformance_suite.h \
binary_json_conformance_suite.cc \
conformance_test_runner.cc \
third_party/jsoncpp/json.h \
third_party/jsoncpp/jsoncpp.cpp
......
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "binary_json_conformance_suite.h"
#include "conformance_test.h"
int main(int argc, char *argv[]) {
google::protobuf::BinaryAndJsonConformanceSuite suite;
return google::protobuf::ForkPipeRunner::Run(argc, argv, &suite);
}
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#ifndef CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H
#define CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H
#include "conformance_test.h"
#include "third_party/jsoncpp/json.h"
namespace google {
namespace protobuf {
class BinaryAndJsonConformanceSuite : public ConformanceTestSuite {
public:
BinaryAndJsonConformanceSuite() {}
private:
void RunSuiteImpl();
void RunValidJsonTest(const string& test_name,
ConformanceLevel level,
const string& input_json,
const string& equivalent_text_format);
void RunValidJsonTestWithProtobufInput(
const string& test_name,
ConformanceLevel level,
const protobuf_test_messages::proto3::TestAllTypesProto3& input,
const string& equivalent_text_format);
void RunValidJsonIgnoreUnknownTest(
const string& test_name, ConformanceLevel level, const string& input_json,
const string& equivalent_text_format);
void RunValidProtobufTest(const string& test_name, ConformanceLevel level,
const string& input_protobuf,
const string& equivalent_text_format,
bool is_proto3);
void RunValidBinaryProtobufTest(const string& test_name,
ConformanceLevel level,
const string& input_protobuf,
bool is_proto3);
void RunValidProtobufTestWithMessage(
const string& test_name, ConformanceLevel level,
const Message *input,
const string& equivalent_text_format,
bool is_proto3);
bool ParseJsonResponse(
const conformance::ConformanceResponse& response,
Message* test_message);
bool ParseResponse(
const conformance::ConformanceResponse& response,
const ConformanceRequestSetting& setting,
Message* test_message) override;
typedef std::function<bool(const Json::Value&)> Validator;
void RunValidJsonTestWithValidator(const string& test_name,
ConformanceLevel level,
const string& input_json,
const Validator& validator);
void ExpectParseFailureForJson(const string& test_name,
ConformanceLevel level,
const string& input_json);
void ExpectSerializeFailureForJson(const string& test_name,
ConformanceLevel level,
const string& text_format);
void ExpectParseFailureForProtoWithProtoVersion (const string& proto,
const string& test_name,
ConformanceLevel level,
bool is_proto3);
void ExpectParseFailureForProto(const std::string& proto,
const std::string& test_name,
ConformanceLevel level);
void ExpectHardParseFailureForProto(const std::string& proto,
const std::string& test_name,
ConformanceLevel level);
void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type);
void TestIllegalTags();
template <class MessageType>
void TestOneofMessage (MessageType &message,
bool is_proto3);
template <class MessageType>
void TestUnknownMessage (MessageType &message,
bool is_proto3);
void TestValidDataForType(
google::protobuf::FieldDescriptor::Type,
std::vector<std::pair<std::string, std::string>> values);
std::unique_ptr<google::protobuf::util::TypeResolver>
type_resolver_;
std::string type_url_;
};
} // namespace protobuf
} // namespace google
#endif // CONFORMANCE_BINARY_JSON_CONFORMANCE_SUITE_H
......@@ -55,6 +55,7 @@ enum WireFormat {
UNSPECIFIED = 0;
PROTOBUF = 1;
JSON = 2;
JSPB = 3; // Google internal only. Opensource testees just skip it.
}
enum TestCategory {
......@@ -67,6 +68,8 @@ enum TestCategory {
// https://developers.google.com/protocol-buffers/docs/proto3#json_options
// for more detail.
JSON_IGNORE_UNKNOWN_PARSING_TEST = 3;
JSPB_TEST = 4; // Test jspb wire format. Google internal only.
// Opensource testees just skip it.
}
// Represents a single test case's input. The testee should:
......@@ -85,6 +88,8 @@ message ConformanceRequest {
oneof payload {
bytes protobuf_payload = 1;
string json_payload = 2;
string jspb_payload = 7; // Google internal only.
// Opensource testees just skip it.
}
// Which format should the testee serialize its message to?
......@@ -99,6 +104,9 @@ message ConformanceRequest {
// spedific support in testee programs. Refer to the defintion of TestCategory
// for more information.
TestCategory test_category = 5;
// Specify details for how to encode jspb.
JspbEncodingConfig jspb_encoding_options = 6;
}
// Represents a single test case's output.
......@@ -132,5 +140,16 @@ message ConformanceResponse {
// For when the testee skipped the test, likely because a certain feature
// wasn't supported, like JSON input/output.
string skipped = 5;
// If the input was successfully parsed and the requested output was JSPB,
// serialize to JSPB and set it in this field. JSPB is google internal only
// format. Opensource testees can just skip it.
string jspb_payload = 7;
}
}
// Encoding options for jspb format.
message JspbEncodingConfig {
// Encode the value field of Any as jspb array if ture, otherwise binary.
bool use_jspb_array_any_format = 1;
}
......@@ -137,6 +137,11 @@ void DoTest(const ConformanceRequest& request, ConformanceResponse* response) {
case ConformanceRequest::PAYLOAD_NOT_SET:
GOOGLE_LOG(FATAL) << "Request didn't have payload.";
break;
default:
GOOGLE_LOG(FATAL) << "unknown payload type: "
<< request.payload_case();
break;
}
switch (request.requested_output_format()) {
......
......@@ -3,6 +3,7 @@
require_once("Conformance/WireFormat.php");
require_once("Conformance/ConformanceResponse.php");
require_once("Conformance/ConformanceRequest.php");
require_once("Conformance/JspbEncodingConfig.php");
require_once("Conformance/TestCategory.php");
require_once("Protobuf_test_messages/Proto3/ForeignMessage.php");
require_once("Protobuf_test_messages/Proto3/ForeignEnum.php");
......
......@@ -80,6 +80,11 @@ ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
break;
}
case conformance::JSPB: {
request_.set_jspb_payload(input);
break;
}
default:
GOOGLE_LOG(FATAL) << "Unspecified input format";
}
......@@ -215,23 +220,27 @@ void ConformanceTestSuite::RunValidInputTest(
void ConformanceTestSuite::RunValidBinaryInputTest(
const ConformanceRequestSetting& setting,
const string& equivalent_wire_format) {
const ConformanceRequest& request = setting.GetRequest();
ConformanceResponse response;
RunTest(setting.GetTestName(), request, &response);
VerifyResponse(setting, equivalent_wire_format, response, true);
}
void ConformanceTestSuite::VerifyResponse(
const ConformanceRequestSetting& setting,
const string& equivalent_wire_format,
const ConformanceResponse& response,
bool need_report_success) {
Message* test_message = setting.GetTestMessage();
const ConformanceRequest& request = setting.GetRequest();
const string& test_name = setting.GetTestName();
ConformanceLevel level = setting.GetLevel();
Message* reference_message = setting.GetTestMessage();
GOOGLE_CHECK(
reference_message->ParseFromString(equivalent_wire_format))
<< "Failed to parse wire data for test case: " << test_name;
const ConformanceRequest& request = setting.GetRequest();
ConformanceResponse response;
RunTest(test_name, request, &response);
Message* test_message = setting.GetTestMessage();
WireFormat requested_output = request.requested_output_format();
switch (response.result_case()) {
case ConformanceResponse::RESULT_NOT_SET:
ReportFailure(test_name, level, request, response,
......@@ -249,53 +258,8 @@ void ConformanceTestSuite::RunValidBinaryInputTest(
ReportSkip(test_name, request, response);
return;
case ConformanceResponse::kJsonPayload: {
if (requested_output != conformance::JSON) {
ReportFailure(
test_name, level, request, response,
"Test was asked for protobuf output but provided JSON instead.");
return;
}
string binary_protobuf;
Status status =
JsonToBinaryString(type_resolver_.get(), type_url_,
response.json_payload(), &binary_protobuf);
if (!status.ok()) {
ReportFailure(test_name, level, request, response,
"JSON output we received from test was unparseable.");
return;
}
if (!test_message->ParseFromString(binary_protobuf)) {
ReportFailure(test_name, level, request, response,
"INTERNAL ERROR: internal JSON->protobuf transcode "
"yielded unparseable proto.");
return;
}
break;
}
case ConformanceResponse::kProtobufPayload: {
if (requested_output != conformance::PROTOBUF) {
ReportFailure(
test_name, level, request, response,
"Test was asked for JSON output but provided protobuf instead.");
return;
}
if (!test_message->ParseFromString(response.protobuf_payload())) {
ReportFailure(test_name, level, request, response,
"Protobuf output we received from test was unparseable.");
return;
}
break;
}
default:
GOOGLE_LOG(FATAL) << test_name << ": unknown payload type: "
<< response.result_case();
if (!ParseResponse(response, setting, test_message)) return;
}
MessageDifferencer differencer;
......@@ -308,7 +272,9 @@ void ConformanceTestSuite::RunValidBinaryInputTest(
bool check;
check = differencer.Compare(*reference_message, *test_message);
if (check) {
ReportSuccess(test_name);
if (need_report_success) {
ReportSuccess(test_name);
}
} else {
ReportFailure(test_name, level, request, response,
"Output was not equivalent to reference message: %s.",
......@@ -375,6 +341,24 @@ bool ConformanceTestSuite::CheckSetEmpty(
}
}
string ConformanceTestSuite::WireFormatToString(
WireFormat wire_format) {
switch (wire_format) {
case conformance::PROTOBUF:
return "PROTOBUF";
case conformance::JSON:
return "JSON";
case conformance::JSPB:
return "JSPB";
case conformance::UNSPECIFIED:
return "UNSPECIFIED";
default:
GOOGLE_LOG(FATAL) << "unknown wire type: "
<< wire_format;
}
return "";
}
bool ConformanceTestSuite::RunSuite(
ConformanceTestRunner* runner, std::string* output) {
runner_ = runner;
......
......@@ -87,7 +87,6 @@ class ForkPipeRunner : public ConformanceTestRunner {
static int Run(int argc, char *argv[],
ConformanceTestSuite* suite);
private:
ForkPipeRunner(const std::string &executable)
: child_pid_(-1), executable_(executable) {}
......@@ -97,24 +96,7 @@ class ForkPipeRunner : public ConformanceTestRunner {
const std::string& request,
std::string* response);
// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
// src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
// of tests and subprocesses like java that can take 100ms or more to start
// up.
//
// 2. Sending all the tests in one big message and receiving all results in one
// big message would take away our visibility about which test(s) caused a
// crash or other fatal error. It would also give us only a single failure
// instead of all of them.
private:
void SpawnTestProgram();
void CheckedWrite(int fd, const void *buf, size_t len);
......@@ -237,6 +219,7 @@ class ConformanceTestSuite {
protected:
virtual string InputFormatString(conformance::WireFormat format) const;
virtual string OutputFormatString(conformance::WireFormat format) const;
conformance::ConformanceRequest request_;
private:
ConformanceLevel level_;
......@@ -244,11 +227,24 @@ class ConformanceTestSuite {
::conformance::WireFormat output_format_;
const Message& prototype_message_;
string test_name_;
conformance::ConformanceRequest request_;
};
bool CheckSetEmpty(const std::set<string>& set_to_check,
const std::string& write_to_file, const std::string& msg);
string WireFormatToString(conformance::WireFormat wire_format);
// Parse payload in the response to the given message. Returns true on
// success.
virtual bool ParseResponse(
const conformance::ConformanceResponse& response,
const ConformanceRequestSetting& setting,
Message* test_message) = 0;
void VerifyResponse(
const ConformanceRequestSetting& setting,
const string& equivalent_wire_format,
const conformance::ConformanceResponse& response,
bool need_report_success);
void ReportSuccess(const std::string& test_name);
void ReportFailure(const string& test_name,
......@@ -295,10 +291,6 @@ class ConformanceTestSuite {
// The set of tests that the testee opted out of;
std::set<std::string> skipped_;
std::unique_ptr<google::protobuf::util::TypeResolver>
type_resolver_;
std::string type_url_;
};
} // namespace protobuf
......
......@@ -212,6 +212,24 @@ int ForkPipeRunner::Run(
return ok ? EXIT_SUCCESS : EXIT_FAILURE;
}
// TODO(haberman): make this work on Windows, instead of using these
// UNIX-specific APIs.
//
// There is a platform-agnostic API in
// src/google/protobuf/compiler/subprocess.h
//
// However that API only supports sending a single message to the subprocess.
// We really want to be able to send messages and receive responses one at a
// time:
//
// 1. Spawning a new process for each test would take way too long for thousands
// of tests and subprocesses like java that can take 100ms or more to start
// up.
//
// 2. Sending all the tests in one big message and receiving all results in one
// big message would take away our visibility about which test(s) caused a
// crash or other fatal error. It would also give us only a single failure
// instead of all of them.
void ForkPipeRunner::SpawnTestProgram() {
int toproc_pipe_fd[2];
int fromproc_pipe_fd[2];
......
......@@ -22,6 +22,11 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>com.google.errorprone</groupId>
<artifactId>error_prone_annotations</artifactId>
<version>2.3.2</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava-testlib</artifactId>
......
......@@ -41,6 +41,7 @@ class Label
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
......@@ -136,6 +136,7 @@ class Type
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
......@@ -41,6 +41,7 @@ class CType
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
......@@ -45,6 +45,7 @@ class JSType
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
......@@ -47,6 +47,7 @@ class OptimizeMode
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
......@@ -47,6 +47,7 @@ class IdempotencyLevel
return self::$valueToName[$value];
}
public static function value($name)
{
$const = __CLASS__ . '::' . strtoupper($name);
......
This diff is collapsed.
......@@ -208,6 +208,7 @@ libprotobuf_lite_la_SOURCES = \
google/protobuf/generated_message_table_driven_lite.cc \
google/protobuf/implicit_weak_message.cc \
google/protobuf/message_lite.cc \
google/protobuf/parse_context.cc \
google/protobuf/repeated_field.cc \
google/protobuf/wire_format_lite.cc \
google/protobuf/io/coded_stream.cc \
......
......@@ -431,7 +431,11 @@ class BigEndian {
} // namespace protobuf
} // namespace google
#ifdef PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
#define GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER 1
#else
#define GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER 0
#endif
#include <google/protobuf/port_undef.inc>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment