Merge pull request #2861 from byronyi/#710

migrate delimited messages functions to util package

BUILD

@@ -145,6 +145,7 @@ cc_library(
         "src/google/protobuf/timestamp.pb.cc",
         "src/google/protobuf/type.pb.cc",
         "src/google/protobuf/unknown_field_set.cc",
+        "src/google/protobuf/util/delimited_message_util.cc",
         "src/google/protobuf/util/field_comparator.cc",
         "src/google/protobuf/util/field_mask_util.cc",
         "src/google/protobuf/util/internal/datapiece.cc",
@@ -521,6 +522,7 @@ cc_test(
         "src/google/protobuf/stubs/type_traits_unittest.cc",
         "src/google/protobuf/text_format_unittest.cc",
         "src/google/protobuf/unknown_field_set_unittest.cc",
+        "src/google/protobuf/util/delimited_message_util_test.cc",
         "src/google/protobuf/util/field_comparator_test.cc",
         "src/google/protobuf/util/field_mask_util_test.cc",
         "src/google/protobuf/util/internal/default_value_objectwriter_test.cc",

cmake/libprotobuf.cmake

@@ -30,6 +30,7 @@ set(libprotobuf_files
   ${protobuf_source_dir}/src/google/protobuf/timestamp.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/type.pb.cc
   ${protobuf_source_dir}/src/google/protobuf/unknown_field_set.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_comparator.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util.cc
   ${protobuf_source_dir}/src/google/protobuf/util/internal/datapiece.cc

cmake/tests.cmake

@@ -169,6 +169,7 @@ set(tests_files
   ${protobuf_source_dir}/src/google/protobuf/stubs/type_traits_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/text_format_unittest.cc
   ${protobuf_source_dir}/src/google/protobuf/unknown_field_set_unittest.cc
+  ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util_test.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_comparator_test.cc
   ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util_test.cc
   ${protobuf_source_dir}/src/google/protobuf/util/internal/default_value_objectwriter_test.cc

src/Makefile.am

@@ -169,6 +169,7 @@ nobase_include_HEADERS =                                         \
   google/protobuf/compiler/python/python_generator.h             \
   google/protobuf/compiler/ruby/ruby_generator.h                 \
   google/protobuf/util/type_resolver.h                           \
+  google/protobuf/util/delimited_message_util.h                  \
   google/protobuf/util/field_comparator.h                        \
   google/protobuf/util/field_mask_util.h                         \
   google/protobuf/util/json_util.h                               \
@@ -267,6 +268,7 @@ libprotobuf_la_SOURCES =                                       \
   google/protobuf/io/zero_copy_stream_impl.cc                  \
   google/protobuf/compiler/importer.cc                         \
   google/protobuf/compiler/parser.cc                           \
+  google/protobuf/util/delimited_message_util.cc               \
   google/protobuf/util/field_comparator.cc                     \
   google/protobuf/util/field_mask_util.cc                      \
   google/protobuf/util/internal/constants.h                    \
@@ -800,6 +802,7 @@ protobuf_test_SOURCES =                                        \
   google/protobuf/compiler/ruby/ruby_generator_unittest.cc     \
   google/protobuf/compiler/csharp/csharp_bootstrap_unittest.cc \
   google/protobuf/compiler/csharp/csharp_generator_unittest.cc \
+  google/protobuf/util/delimited_message_util_test.cc          \
   google/protobuf/util/field_comparator_test.cc                \
   google/protobuf/util/field_mask_util_test.cc                 \
   google/protobuf/util/internal/default_value_objectwriter_test.cc \

src/google/protobuf/util/delimited_message_util.cc

+// Adapted from the patch of kenton@google.com (Kenton Varda)
+// See https://github.com/google/protobuf/pull/710 for details.
+
+#include <google/protobuf/util/delimited_message_util.h>
+
+namespace google {
+namespace protobuf {
+namespace util {
+
+bool SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor) {
+  io::FileOutputStream output(file_descriptor);
+  return SerializeDelimitedToZeroCopyStream(message, &output);
+}
+
+bool SerializeDelimitedToOstream(const MessageLite& message, ostream* output) {
+  {
+    io::OstreamOutputStream zero_copy_output(output);
+    if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false;
+  }
+  return output->good();
+}
+
+bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof) {
+  google::protobuf::io::CodedInputStream coded_input(input);
+  return ParseDelimitedFromCodedStream(message, &coded_input, clean_eof);
+}
+
+bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof) {
+  if (clean_eof != NULL) *clean_eof = false;
+  int start = input->CurrentPosition();
+
+  // Read the size.
+  uint32 size;
+  if (!input->ReadVarint32(&size)) {
+    if (clean_eof != NULL) *clean_eof = input->CurrentPosition() == start;
+    return false;
+  }
+
+  // Tell the stream not to read beyond that size.
+  google::protobuf::io::CodedInputStream::Limit limit = input->PushLimit(size);
+
+  // Parse the message.
+  if (!message->MergeFromCodedStream(input)) return false;
+  if (!input->ConsumedEntireMessage()) return false;
+
+  // Release the limit.
+  input->PopLimit(limit);
+
+  return true;
+}
+
+bool SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output) {
+  google::protobuf::io::CodedOutputStream coded_output(output);
+  return SerializeDelimitedToCodedStream(message, &coded_output);
+}
+
+bool SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output) {
+  // Write the size.
+  int size = message.ByteSize();
+  output->WriteVarint32(size);
+
+  // Write the content.
+  uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size);
+  if (buffer != NULL) {
+    // Optimization: The message fits in one buffer, so use the faster
+    // direct-to-array serialization path.
+    message.SerializeWithCachedSizesToArray(buffer);
+  } else {
+    // Slightly-slower path when the message is multiple buffers.
+    message.SerializeWithCachedSizes(output);
+    if (output->HadError()) return false;
+  }
+
+  return true;
+}
+
+}  // namespace util
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/util/delimited_message_util.h

+// Adapted from the patch of kenton@google.com (Kenton Varda)
+// See https://github.com/google/protobuf/pull/710 for details.
+
+#ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__
+#define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__
+
+#include <ostream>
+
+#include <google/protobuf/message_lite.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+
+namespace google {
+namespace protobuf {
+namespace util {
+
+// Write a single size-delimited message from the given stream. Delimited
+// format allows a single file or stream to contain multiple messages,
+// whereas normally writing multiple non-delimited messages to the same
+// stream would cause them to be merged. A delimited message is a varint
+// encoding the message size followed by a message of exactly that size.
+//
+// Note that if you want to *read* a delimited message from a file descriptor
+// or istream, you will need to construct an io::FileInputStream or
+// io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the
+// utility function ParseDelimitedFromZeroCopyStream(). You must then
+// continue to use the same ZeroCopyInputStream to read all further data from
+// the stream until EOF. This is because these ZeroCopyInputStream
+// implementations are buffered: they read a big chunk of data at a time,
+// then parse it. As a result, they may read past the end of the delimited
+// message. There is no way for them to push the extra data back into the
+// underlying source, so instead you must keep using the same stream object.
+bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor);
+
+bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const MessageLite& message, ostream* output);
+
+// Read a single size-delimited message from the given stream. Delimited
+// format allows a single file or stream to contain multiple messages,
+// whereas normally parsing consumes the entire input. A delimited message
+// is a varint encoding the message size followed by a message of exactly
+// that size.
+//
+// If |clean_eof| is not NULL, then it will be set to indicate whether the
+// stream ended cleanly. That is, if the stream ends without this method
+// having read any data at all from it, then *clean_eof will be set true,
+// otherwise it will be set false. Note that these methods return false
+// on EOF, but they also return false on other errors, so |clean_eof| is
+// needed to distinguish a clean end from errors.
+bool LIBPROTOBUF_EXPORT ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof);
+
+bool LIBPROTOBUF_EXPORT ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof);
+
+// Write a single size-delimited message from the given stream. Delimited
+// format allows a single file or stream to contain multiple messages,
+// whereas normally writing multiple non-delimited messages to the same
+// stream would cause them to be merged. A delimited message is a varint
+// encoding the message size followed by a message of exactly that size.
+bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output);
+
+bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output);
+
+}  // namespace util
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__

src/google/protobuf/util/delimited_message_util_test.cc

+// Adapted from the patch of kenton@google.com (Kenton Varda)
+// See https://github.com/google/protobuf/pull/710 for details.
+
+#include <google/protobuf/util/delimited_message_util.h>
+
+#include <sstream>
+
+#include <google/protobuf/test_util.h>
+#include <google/protobuf/unittest.pb.h>
+#include <google/protobuf/testing/googletest.h>
+#include <gtest/gtest.h>
+
+namespace google {
+namespace protobuf {
+namespace util {
+
+TEST(DelimitedMessageUtilTest, DelimitedMessages) {
+  stringstream stream;
+
+  {
+    protobuf_unittest::TestAllTypes message1;
+    TestUtil::SetAllFields(&message1);
+    EXPECT_TRUE(SerializeDelimitedToOstream(message1, &stream));
+
+    protobuf_unittest::TestPackedTypes message2;
+    TestUtil::SetPackedFields(&message2);
+    EXPECT_TRUE(SerializeDelimitedToOstream(message2, &stream));
+  }
+
+  {
+    bool clean_eof;
+    io::IstreamInputStream zstream(&stream);
+
+    protobuf_unittest::TestAllTypes message1;
+    clean_eof = true;
+    EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message1,
+        &zstream, &clean_eof));
+    EXPECT_FALSE(clean_eof);
+    TestUtil::ExpectAllFieldsSet(message1);
+
+    protobuf_unittest::TestPackedTypes message2;
+    clean_eof = true;
+    EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message2,
+        &zstream, &clean_eof));
+    EXPECT_FALSE(clean_eof);
+    TestUtil::ExpectPackedFieldsSet(message2);
+
+    clean_eof = false;
+    EXPECT_FALSE(ParseDelimitedFromZeroCopyStream(&message2,
+        &zstream, &clean_eof));
+    EXPECT_TRUE(clean_eof);
+  }
+}
+
+}  // namespace util
+}  // namespace protobuf
+}  // namespace google