Skip to content

P
protobuf
Commit a269a6d3 authored by Bo Yang's avatar Bo Yang
Browse files
Options

Implement parsing for proto3 primitive repeated fields. Previously, for 

proto3 primitive repeated fields, packed data cannot be parsed if
definition is unpacked. Neither is the other way.
parent 4cbb6122
Hide whitespace changes
Inline Side-by-side
Showing with 94 additions and 62 deletions
src/google/protobuf/compiler/cpp/cpp_message.cc
View file @ a269a6d3
...@@ -2932,7 +2932,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { ...@@ -2932,7 +2932,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
"commontag", SimpleItoa(WireFormat::MakeTag(field))); "commontag", SimpleItoa(WireFormat::MakeTag(field)));
if (need_label || if (need_label ||
(field->is_repeated() && !field->options().packed() && !loops)) { (field->is_repeated() && !field->is_packed() && !loops)) {
printer->Print( printer->Print(
" parse_$name$:\n", " parse_$name$:\n",
"name", field->name()); "name", field->name());
...@@ -2945,7 +2945,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { ...@@ -2945,7 +2945,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
} }
printer->Indent(); printer->Indent();
if (field->options().packed()) { if (field->is_packed()) {
field_generator.GenerateMergeFromCodedStreamWithPacking(printer); field_generator.GenerateMergeFromCodedStreamWithPacking(printer);
} else { } else {
field_generator.GenerateMergeFromCodedStream(printer); field_generator.GenerateMergeFromCodedStream(printer);
...@@ -2953,7 +2953,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { ...@@ -2953,7 +2953,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
printer->Outdent(); printer->Outdent();
// Emit code to parse unexpectedly packed or unpacked values. // Emit code to parse unexpectedly packed or unpacked values.
if (field->is_packable() && field->options().packed()) { if (field->is_packed()) {
internal::WireFormatLite::WireType wiretype = internal::WireFormatLite::WireType wiretype =
WireFormat::WireTypeForFieldType(field->type()); WireFormat::WireTypeForFieldType(field->type());
printer->Print("} else if (tag == $uncommontag$) {\n", printer->Print("} else if (tag == $uncommontag$) {\n",
...@@ -2963,7 +2963,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { ...@@ -2963,7 +2963,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
printer->Indent(); printer->Indent();
field_generator.GenerateMergeFromCodedStream(printer); field_generator.GenerateMergeFromCodedStream(printer);
printer->Outdent(); printer->Outdent();
} else if (field->is_packable() && !field->options().packed()) { } else if (field->is_packable() && !field->is_packed()) {
internal::WireFormatLite::WireType wiretype = internal::WireFormatLite::WireType wiretype =
internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED; internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
printer->Print("} else if (tag == $uncommontag$) {\n", printer->Print("} else if (tag == $uncommontag$) {\n",
...@@ -2988,7 +2988,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) { ...@@ -2988,7 +2988,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
"if (input->ExpectTag($tag$)) goto parse_loop_$name$;\n", "if (input->ExpectTag($tag$)) goto parse_loop_$name$;\n",
"tag", SimpleItoa(WireFormat::MakeTag(field)), "tag", SimpleItoa(WireFormat::MakeTag(field)),
"name", field->name()); "name", field->name());
} else if (field->is_repeated() && !field->options().packed()) { } else if (field->is_repeated() && !field->is_packed()) {
printer->Print( printer->Print(
"if (input->ExpectTag($tag$)) goto parse_$name$;\n", "if (input->ExpectTag($tag$)) goto parse_$name$;\n",
"tag", SimpleItoa(WireFormat::MakeTag(field)), "tag", SimpleItoa(WireFormat::MakeTag(field)),
......
src/google/protobuf/wire_format.h
View file @ a269a6d3
...@@ -290,7 +290,7 @@ class LIBPROTOBUF_EXPORT UnknownFieldSetFieldSkipper : public FieldSkipper { ...@@ -290,7 +290,7 @@ class LIBPROTOBUF_EXPORT UnknownFieldSetFieldSkipper : public FieldSkipper {
inline WireFormatLite::WireType WireFormat::WireTypeForField( inline WireFormatLite::WireType WireFormat::WireTypeForField(
const FieldDescriptor* field) { const FieldDescriptor* field) {
if (field->options().packed()) { if (field->is_packed()) {
return WireFormatLite::WIRETYPE_LENGTH_DELIMITED; return WireFormatLite::WIRETYPE_LENGTH_DELIMITED;
} else { } else {
return WireTypeForFieldType(field->type()); return WireTypeForFieldType(field->type());
......
src/google/protobuf/wire_format_unittest.cc
View file @ a269a6d3
...@@ -795,9 +795,73 @@ TEST(WireFormatTest, CompatibleTypes) { ...@@ -795,9 +795,73 @@ TEST(WireFormatTest, CompatibleTypes) {
ASSERT_EQ(static_cast<uint32>(data), msg5.data()); ASSERT_EQ(static_cast<uint32>(data), msg5.data());
} }
class Proto3PrimitiveRepeatedWireFormatTest class Proto3PrimitiveRepeatedWireFormatTest : public ::testing::Test {
: public ::testing::TestWithParam<bool> {
protected: protected:
Proto3PrimitiveRepeatedWireFormatTest()
: packedTestAllTypes_(
"\xFA\x01\x01\x01"
"\x82\x02\x01\x01"
"\x8A\x02\x01\x01"
"\x92\x02\x01\x01"
"\x9A\x02\x01\x02"
"\xA2\x02\x01\x02"
"\xAA\x02\x04\x01\x00\x00\x00"
"\xB2\x02\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\xBA\x02\x04\x01\x00\x00\x00"
"\xC2\x02\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\xCA\x02\x04\x00\x00\x80\x3f"
"\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\xDA\x02\x01\x01"
"\x9A\x03\x01\x01",
86),
packedTestUnpackedTypes_(
"\x0A\x01\x01"
"\x12\x01\x01"
"\x1A\x01\x01"
"\x22\x01\x01"
"\x2A\x01\x02"
"\x32\x01\x02"
"\x3A\x04\x01\x00\x00\x00"
"\x42\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\x4A\x04\x01\x00\x00\x00"
"\x52\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\x5A\x04\x00\x00\x80\x3f"
"\x62\x08\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\x6A\x01\x01"
"\x72\x01\x01",
72),
unpackedTestAllTypes_(
"\xF8\x01\x01"
"\x80\x02\x01"
"\x88\x02\x01"
"\x90\x02\x01"
"\x98\x02\x02"
"\xA0\x02\x02"
"\xAD\x02\x01\x00\x00\x00"
"\xB1\x02\x01\x00\x00\x00\x00\x00\x00\x00"
"\xBD\x02\x01\x00\x00\x00"
"\xC1\x02\x01\x00\x00\x00\x00\x00\x00\x00"
"\xCD\x02\x00\x00\x80\x3f"
"\xD1\x02\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\xD8\x02\x01"
"\x98\x03\x01",
72),
unpackedTestUnpackedTypes_(
"\x08\x01"
"\x10\x01"
"\x18\x01"
"\x20\x01"
"\x28\x02"
"\x30\x02"
"\x3D\x01\x00\x00\x00"
"\x41\x01\x00\x00\x00\x00\x00\x00\x00"
"\x4D\x01\x00\x00\x00"
"\x51\x01\x00\x00\x00\x00\x00\x00\x00"
"\x5D\x00\x00\x80\x3f"
"\x61\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\x68\x01"
"\x70\x01",
58) {}
template <class Proto> template <class Proto>
void SetProto3PrimitiveRepeatedFields(Proto* message) { void SetProto3PrimitiveRepeatedFields(Proto* message) {
message->add_repeated_int32(1); message->add_repeated_int32(1);
...@@ -837,8 +901,7 @@ class Proto3PrimitiveRepeatedWireFormatTest ...@@ -837,8 +901,7 @@ class Proto3PrimitiveRepeatedWireFormatTest
} }
template <class Proto> template <class Proto>
void TestProto3PrimitiveRepeatedFields(Proto* message, void TestSerialization(Proto* message, const string& expected) {
const string& expected) {
SetProto3PrimitiveRepeatedFields(message); SetProto3PrimitiveRepeatedFields(message);
int size = message->ByteSize(); int size = message->ByteSize();
...@@ -851,13 +914,8 @@ class Proto3PrimitiveRepeatedWireFormatTest ...@@ -851,13 +914,8 @@ class Proto3PrimitiveRepeatedWireFormatTest
message->SerializeWithCachedSizes(&output); message->SerializeWithCachedSizes(&output);
ASSERT_FALSE(output.HadError()); ASSERT_FALSE(output.HadError());
} }
EXPECT_TRUE(expected == generated_data); EXPECT_TRUE(expected == generated_data);
message->Clear();
message->ParseFromString(generated_data);
ExpectProto3PrimitiveRepeatedFieldsSet(*message);
// Serialize using the dynamic code. // Serialize using the dynamic code.
string dynamic_data; string dynamic_data;
{ {
...@@ -866,64 +924,38 @@ class Proto3PrimitiveRepeatedWireFormatTest ...@@ -866,64 +924,38 @@ class Proto3PrimitiveRepeatedWireFormatTest
WireFormat::SerializeWithCachedSizes(*message, size, &output); WireFormat::SerializeWithCachedSizes(*message, size, &output);
ASSERT_FALSE(output.HadError()); ASSERT_FALSE(output.HadError());
} }
EXPECT_TRUE(expected == dynamic_data); EXPECT_TRUE(expected == dynamic_data);
}
template <class Proto>
void TestParsing(Proto* message, const string& compatible_data) {
message->Clear();
message->ParseFromString(compatible_data);
ExpectProto3PrimitiveRepeatedFieldsSet(*message);
message->Clear(); message->Clear();
io::CodedInputStream input( io::CodedInputStream input(
reinterpret_cast<const uint8*>(dynamic_data.data()), reinterpret_cast<const uint8*>(compatible_data.data()),
dynamic_data.size()); compatible_data.size());
WireFormat::ParseAndMergePartial(&input, message); WireFormat::ParseAndMergePartial(&input, message);
ExpectProto3PrimitiveRepeatedFieldsSet(*message); ExpectProto3PrimitiveRepeatedFieldsSet(*message);
} }
const string packedTestAllTypes_;
const string packedTestUnpackedTypes_;
const string unpackedTestAllTypes_;
const string unpackedTestUnpackedTypes_;
}; };
INSTANTIATE_TEST_CASE_P(SetPacked,
Proto3PrimitiveRepeatedWireFormatTest,
::testing::Values(false, true));
TEST_P(Proto3PrimitiveRepeatedWireFormatTest, Proto3PrimitiveRepeated) { TEST_F(Proto3PrimitiveRepeatedWireFormatTest, Proto3PrimitiveRepeated) {
proto3_arena_unittest::TestAllTypes packed_message; proto3_arena_unittest::TestAllTypes packed_message;
proto3_arena_unittest::TestUnpackedTypes unpacked_message; proto3_arena_unittest::TestUnpackedTypes unpacked_message;
TestSerialization(&packed_message, packedTestAllTypes_);
const string packedExpected( TestParsing(&packed_message, packedTestAllTypes_);
"\xFA\x01\x01\x01" TestParsing(&packed_message, unpackedTestAllTypes_);
"\x82\x02\x01\x01" TestSerialization(&unpacked_message, unpackedTestUnpackedTypes_);
"\x8A\x02\x01\x01" TestParsing(&unpacked_message, packedTestUnpackedTypes_);
"\x92\x02\x01\x01" TestParsing(&unpacked_message, unpackedTestUnpackedTypes_);
"\x9A\x02\x01\x02"
"\xA2\x02\x01\x02"
"\xAA\x02\x04\x01\x00\x00\x00"
"\xB2\x02\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\xBA\x02\x04\x01\x00\x00\x00"
"\xC2\x02\x08\x01\x00\x00\x00\x00\x00\x00\x00"
"\xCA\x02\x04\x00\x00\x80\x3f"
"\xD2\x02\x08\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\xDA\x02\x01\x01"
"\x9A\x03\x01\x01",
86);
const string unpackedExpected(
"\x08\x01"
"\x10\x01"
"\x18\x01"
"\x20\x01"
"\x28\x02"
"\x30\x02"
"\x3D\x01\x00\x00\x00"
"\x41\x01\x00\x00\x00\x00\x00\x00\x00"
"\x4D\x01\x00\x00\x00"
"\x51\x01\x00\x00\x00\x00\x00\x00\x00"
"\x5D\x00\x00\x80\x3f"
"\x61\x00\x00\x00\x00\x00\x00\xf0\x3f"
"\x68\x01"
"\x70\x01",
58);
if (GetParam()) {
TestProto3PrimitiveRepeatedFields(&packed_message, packedExpected);
} else {
TestProto3PrimitiveRepeatedFields(&unpacked_message, unpackedExpected);
}
} }
class WireFormatInvalidInputTest : public testing::Test { class WireFormatInvalidInputTest : public testing::Test {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment