Integrate recent changes from google3.

protoc
- New flags --encode and --decode can be used to convert between protobuf text
  format and binary format from the command-line.
- New flag --descriptor_set_out can be used to write FileDescriptorProtos for
  all parsed files directly into a single output file.  This is particularly
  useful if you wish to parse .proto files from programs written in languages
  other than C++: just run protoc as a background process and have it output
  a FileDescriptorList, then parse that natively.

C++
- Reflection objects are now per-class rather than per-instance.  To make this
  possible, the Reflection interface had to be changed such that all methods
  take the Message instance as a parameter.  This change improves performance
  significantly in memory-bandwidth-limited use cases, since it makes the
  message objects smaller.  Note that source-incompatible interface changes
  like this will not be made again after the library leaves beta.

Python
- MergeFrom(message) and CopyFrom(message) are now implemented.
- SerializeToString() raises an exception if the message is missing required
  fields.
- Code organization improvements.
- Fixed doc comments for RpcController and RpcChannel, which had somehow been
  swapped.

generate_descriptor_proto.sh

@@ -6,14 +6,6 @@
 # will fail if these files do not match what the protocol compiler would
 # generate.
 
-# Note that this will always need to be run once after running
-# extract_from_google3.sh.  That script initially copies descriptor.pb.{h,cc}
-# over from the google3 code and fixes it up to compile outside of google3, but
-# it cannot fix the encoded descriptor embedded in descriptor.pb.cc.  So, once
-# the protocol compiler has been built with the slightly-broken
-# descriptor.pb.cc, the files must be regenerated and the compiler must be
-# built again.
-
 if test ! -e src/google/protobuf/stubs/common.h; then
   cat >&2 << __EOF__
 Could not find source code.  Make sure you are running this script from the

python/google/protobuf/internal/reflection_test.py

@@ -656,9 +656,172 @@ class RefectionTest(unittest.TestCase):
     self.assertRaises(KeyError, extendee_proto.HasExtension,
                       unittest_pb2.repeated_string_extension)
 
-  def testCopyFrom(self):
-    # TODO(robinson): Implement.
-    pass
+  def testMergeFromSingularField(self):
+    # Test merge with just a singular field.
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.optional_int32 = 1
+
+    proto2 = unittest_pb2.TestAllTypes()
+    # This shouldn't get overwritten.
+    proto2.optional_string = 'value'
+
+    proto2.MergeFrom(proto1)
+    self.assertEqual(1, proto2.optional_int32)
+    self.assertEqual('value', proto2.optional_string)
+
+  def testMergeFromRepeatedField(self):
+    # Test merge with just a repeated field.
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.repeated_int32.append(1)
+    proto1.repeated_int32.append(2)
+
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.repeated_int32.append(0)
+    proto2.MergeFrom(proto1)
+
+    self.assertEqual(0, proto2.repeated_int32[0])
+    self.assertEqual(1, proto2.repeated_int32[1])
+    self.assertEqual(2, proto2.repeated_int32[2])
+
+  def testMergeFromOptionalGroup(self):
+    # Test merge with an optional group.
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.optionalgroup.a = 12
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.MergeFrom(proto1)
+    self.assertEqual(12, proto2.optionalgroup.a)
+
+  def testMergeFromRepeatedNestedMessage(self):
+    # Test merge with a repeated nested message.
+    proto1 = unittest_pb2.TestAllTypes()
+    m = proto1.repeated_nested_message.add()
+    m.bb = 123
+    m = proto1.repeated_nested_message.add()
+    m.bb = 321
+
+    proto2 = unittest_pb2.TestAllTypes()
+    m = proto2.repeated_nested_message.add()
+    m.bb = 999
+    proto2.MergeFrom(proto1)
+    self.assertEqual(999, proto2.repeated_nested_message[0].bb)
+    self.assertEqual(123, proto2.repeated_nested_message[1].bb)
+    self.assertEqual(321, proto2.repeated_nested_message[2].bb)
+
+  def testMergeFromAllFields(self):
+    # With all fields set.
+    proto1 = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(proto1)
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.MergeFrom(proto1)
+
+    # Messages should be equal.
+    self.assertEqual(proto2, proto1)
+
+    # Serialized string should be equal too.
+    string1 = proto1.SerializeToString()
+    string2 = proto2.SerializeToString()
+    self.assertEqual(string1, string2)
+
+  def testMergeFromExtensionsSingular(self):
+    proto1 = unittest_pb2.TestAllExtensions()
+    proto1.Extensions[unittest_pb2.optional_int32_extension] = 1
+
+    proto2 = unittest_pb2.TestAllExtensions()
+    proto2.MergeFrom(proto1)
+    self.assertEqual(
+        1, proto2.Extensions[unittest_pb2.optional_int32_extension])
+
+  def testMergeFromExtensionsRepeated(self):
+    proto1 = unittest_pb2.TestAllExtensions()
+    proto1.Extensions[unittest_pb2.repeated_int32_extension].append(1)
+    proto1.Extensions[unittest_pb2.repeated_int32_extension].append(2)
+
+    proto2 = unittest_pb2.TestAllExtensions()
+    proto2.Extensions[unittest_pb2.repeated_int32_extension].append(0)
+    proto2.MergeFrom(proto1)
+    self.assertEqual(
+        3, len(proto2.Extensions[unittest_pb2.repeated_int32_extension]))
+    self.assertEqual(
+        0, proto2.Extensions[unittest_pb2.repeated_int32_extension][0])
+    self.assertEqual(
+        1, proto2.Extensions[unittest_pb2.repeated_int32_extension][1])
+    self.assertEqual(
+        2, proto2.Extensions[unittest_pb2.repeated_int32_extension][2])
+
+  def testMergeFromExtensionsNestedMessage(self):
+    proto1 = unittest_pb2.TestAllExtensions()
+    ext1 = proto1.Extensions[
+        unittest_pb2.repeated_nested_message_extension]
+    m = ext1.add()
+    m.bb = 222
+    m = ext1.add()
+    m.bb = 333
+
+    proto2 = unittest_pb2.TestAllExtensions()
+    ext2 = proto2.Extensions[
+        unittest_pb2.repeated_nested_message_extension]
+    m = ext2.add()
+    m.bb = 111
+
+    proto2.MergeFrom(proto1)
+    ext2 = proto2.Extensions[
+        unittest_pb2.repeated_nested_message_extension]
+    self.assertEqual(3, len(ext2))
+    self.assertEqual(111, ext2[0].bb)
+    self.assertEqual(222, ext2[1].bb)
+    self.assertEqual(333, ext2[2].bb)
+
+  def testCopyFromSingularField(self):
+    # Test copy with just a singular field.
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.optional_int32 = 1
+    proto1.optional_string = 'important-text'
+
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.optional_string = 'value'
+
+    proto2.CopyFrom(proto1)
+    self.assertEqual(1, proto2.optional_int32)
+    self.assertEqual('important-text', proto2.optional_string)
+
+  def testCopyFromRepeatedField(self):
+    # Test copy with a repeated field.
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.repeated_int32.append(1)
+    proto1.repeated_int32.append(2)
+
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.repeated_int32.append(0)
+    proto2.CopyFrom(proto1)
+
+    self.assertEqual(1, proto2.repeated_int32[0])
+    self.assertEqual(2, proto2.repeated_int32[1])
+
+  def testCopyFromAllFields(self):
+    # With all fields set.
+    proto1 = unittest_pb2.TestAllTypes()
+    test_util.SetAllFields(proto1)
+    proto2 = unittest_pb2.TestAllTypes()
+    proto2.CopyFrom(proto1)
+
+    # Messages should be equal.
+    self.assertEqual(proto2, proto1)
+
+    # Serialized string should be equal too.
+    string1 = proto1.SerializeToString()
+    string2 = proto2.SerializeToString()
+    self.assertEqual(string1, string2)
+
+  def testCopyFromSelf(self):
+    proto1 = unittest_pb2.TestAllTypes()
+    proto1.repeated_int32.append(1)
+    proto1.optional_int32 = 2
+    proto1.optional_string = 'important-text'
+
+    proto1.CopyFrom(proto1)
+    self.assertEqual(1, proto1.repeated_int32[0])
+    self.assertEqual(2, proto1.optional_int32)
+    self.assertEqual('important-text', proto1.optional_string)
 
   def testClear(self):
     proto = unittest_pb2.TestAllTypes()
@@ -1256,6 +1419,57 @@ class SerializationTest(unittest.TestCase):
     # Parsing this message should succeed.
     proto2.MergeFromString(serialized)
 
+  def _CheckRaises(self, exc_class, callable_obj, exception):
+    """This method checks if the excpetion type and message are as expected."""
+    try:
+      callable_obj()
+    except exc_class, ex:
+      # Check if the exception message is the right one.
+      self.assertEqual(exception, str(ex))
+      return
+    else:
+      raise self.failureException('%s not raised' % str(exc_class))
+
+  def testSerializeUninitialized(self):
+    proto = unittest_pb2.TestRequired()
+    self._CheckRaises(
+        message.EncodeError,
+        proto.SerializeToString,
+        'Required field protobuf_unittest.TestRequired.a is not set.')
+    # Shouldn't raise exceptions.
+    partial = proto.SerializePartialToString()
+
+    proto.a = 1
+    self._CheckRaises(
+        message.EncodeError,
+        proto.SerializeToString,
+        'Required field protobuf_unittest.TestRequired.b is not set.')
+    # Shouldn't raise exceptions.
+    partial = proto.SerializePartialToString()
+
+    proto.b = 2
+    self._CheckRaises(
+        message.EncodeError,
+        proto.SerializeToString,
+        'Required field protobuf_unittest.TestRequired.c is not set.')
+    # Shouldn't raise exceptions.
+    partial = proto.SerializePartialToString()
+
+    proto.c = 3
+    serialized = proto.SerializeToString()
+    # Shouldn't raise exceptions.
+    partial = proto.SerializePartialToString()
+
+    proto2 = unittest_pb2.TestRequired()
+    proto2.MergeFromString(serialized)
+    self.assertEqual(1, proto2.a)
+    self.assertEqual(2, proto2.b)
+    self.assertEqual(3, proto2.c)
+    proto2.ParseFromString(partial)
+    self.assertEqual(1, proto2.a)
+    self.assertEqual(2, proto2.b)
+    self.assertEqual(3, proto2.c)
+
 
 class OptionsTest(unittest.TestCase):
 

python/google/protobuf/message.py

@@ -65,15 +65,43 @@ class Message(object):
     return text_format.MessageToString(self)
 
   def MergeFrom(self, other_msg):
+    """Merges the contents of the specified message into current message.
+
+    This method merges the contents of the specified message into the current
+    message. Singular fields that are set in the specified message overwrite
+    the corresponding fields in the current message. Repeated fields are
+    appended. Singular sub-messages and groups are recursively merged.
+
+    Args:
+      other_msg: Message to merge into the current message.
+    """
     raise NotImplementedError
 
   def CopyFrom(self, other_msg):
-    raise NotImplementedError
+    """Copies the content of the specified message into the current message.
+
+    The method clears the current message and then merges the specified
+    message using MergeFrom.
+
+    Args:
+      other_msg: Message to copy into the current one.
+    """
+    if self == other_msg:
+      return
+    self.Clear()
+    self.MergeFrom(other_msg)
 
   def Clear(self):
+    """Clears all data that was set in the message."""
     raise NotImplementedError
 
   def IsInitialized(self):
+    """Checks if the message is initialized.
+
+    Returns:
+      The method returns True if the message is initialized (i.e. all of its
+      required fields are set).
+    """
     raise NotImplementedError
 
   # TODO(robinson): MergeFromString() should probably return None and be
@@ -118,6 +146,26 @@ class Message(object):
     self.MergeFromString(serialized)
 
   def SerializeToString(self):
+    """Serializes the protocol message to a binary string.
+
+    Returns:
+      A binary string representation of the message if all of the required
+      fields in the message are set (i.e. the message is initialized).
+
+    Raises:
+      message.EncodeError if the message isn't initialized.
+    """
+    raise NotImplementedError
+
+  def SerializePartialToString(self):
+    """Serializes the protocol message to a binary string.
+
+    This method is similar to SerializeToString but doesn't check if the
+    message is initialized.
+
+    Returns:
+      A string representation of the partial message.
+    """
     raise NotImplementedError
 
   # TODO(robinson): Decide whether we like these better

python/google/protobuf/service.py

@@ -85,18 +85,14 @@ class Service(object):
 
 class RpcController(object):
 
-  """Abstract interface for an RPC channel.
-
-  An RpcChannel represents a communication line to a service which can be used
-  to call that service's methods.  The service may be running on another
-  machine. Normally, you should not use an RpcChannel directly, but instead
-  construct a stub {@link Service} wrapping it.  Example:
+  """An RpcController mediates a single method call.
 
-  Example:
-    RpcChannel channel = rpcImpl.Channel("remotehost.example.com:1234")
-    RpcController controller = rpcImpl.Controller()
-    MyService service = MyService_Stub(channel)
-    service.MyMethod(controller, request, callback)
+  The primary purpose of the controller is to provide a way to manipulate
+  settings specific to the RPC implementation and to find out about RPC-level
+  errors. The methods provided by the RpcController interface are intended
+  to be a "least common denominator" set of features which we expect all
+  implementations to support.  Specific implementations may provide more
+  advanced features (e.g. deadline propagation).
   """
 
   # Client-side methods below
@@ -172,14 +168,18 @@ class RpcController(object):
 
 class RpcChannel(object):
 
-  """An RpcController mediates a single method call.
+  """Abstract interface for an RPC channel.
 
-  The primary purpose of the controller is to provide a way to manipulate
-  settings specific to the RPC implementation and to find out about RPC-level
-  errors. The methods provided by the RpcController interface are intended
-  to be a "least common denominator" set of features which we expect all
-  implementations to support.  Specific implementations may provide more
-  advanced features (e.g. deadline propagation).
+  An RpcChannel represents a communication line to a service which can be used
+  to call that service's methods.  The service may be running on another
+  machine. Normally, you should not use an RpcChannel directly, but instead
+  construct a stub {@link Service} wrapping it.  Example:
+
+  Example:
+    RpcChannel channel = rpcImpl.Channel("remotehost.example.com:1234")
+    RpcController controller = rpcImpl.Controller()
+    MyService service = MyService_Stub(channel)
+    service.MyMethod(controller, request, callback)
   """
 
   def CallMethod(self, method_descriptor, rpc_controller,

src/google/protobuf/compiler/command_line_interface.h

@@ -35,6 +35,7 @@ namespace google {
 namespace protobuf {
 
 class FileDescriptor;        // descriptor.h
+class DescriptorPool;        // descriptor.h
 
 namespace compiler {
 
@@ -164,6 +165,12 @@ class LIBPROTOC_EXPORT CommandLineInterface {
   bool GenerateOutput(const FileDescriptor* proto_file,
                       const OutputDirective& output_directive);
 
+  // Implements --encode and --decode.
+  bool EncodeOrDecode(const DescriptorPool* pool);
+
+  // Implements the --descriptor_set_out option.
+  bool WriteDescriptorSet(const vector<const FileDescriptor*> parsed_files);
+
   // -----------------------------------------------------------------
 
   // The name of the executable as invoked (i.e. argv[0]).
@@ -181,6 +188,14 @@ class LIBPROTOC_EXPORT CommandLineInterface {
   GeneratorMap generators_;
 
   // Stuff parsed from command line.
+  enum Mode {
+    MODE_COMPILE,  // Normal mode:  parse .proto files and compile them.
+    MODE_ENCODE,   // --encode:  read text from stdin, write binary to stdout.
+    MODE_DECODE    // --decode:  read binary from stdin, write text to stdout.
+  };
+
+  Mode mode_;
+
   vector<pair<string, string> > proto_path_;  // Search path for proto files.
   vector<string> input_files_;                // Names of the input proto files.
 
@@ -194,6 +209,19 @@ class LIBPROTOC_EXPORT CommandLineInterface {
   };
   vector<OutputDirective> output_directives_;
 
+  // When using --encode or --decode, this names the type we are encoding or
+  // decoding.  (Empty string indicates --decode_raw.)
+  string codec_type_;
+
+  // If --descriptor_set_out was given, this is the filename to which the
+  // FileDescriptorSet should be written.  Otherwise, empty.
+  string descriptor_set_name_;
+
+  // True if --include_imports was given, meaning that we should
+  // write all transitive dependencies to the DescriptorSet.  Otherwise, only
+  // the .proto files listed on the command-line are added.
+  bool imports_in_descriptor_set_;
+
   // Was the --disallow_services flag used?
   bool disallow_services_;
 

src/google/protobuf/compiler/command_line_interface_unittest.cc

@@ -18,13 +18,23 @@
 //  Based on original Protocol Buffers design by
 //  Sanjay Ghemawat, Jeff Dean, and others.
 
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#ifdef _MSC_VER
+#include <io.h>
+#else
+#include <unistd.h>
+#endif
 #include <vector>
 
+#include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/descriptor.h>
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/compiler/command_line_interface.h>
 #include <google/protobuf/compiler/code_generator.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/unittest.pb.h>
 #include <google/protobuf/testing/file.h>
 #include <google/protobuf/stubs/strutil.h>
 
@@ -35,6 +45,15 @@ namespace google {
 namespace protobuf {
 namespace compiler {
 
+#if defined(_WIN32)
+#ifndef STDIN_FILENO
+#define STDIN_FILENO 0
+#endif
+#ifndef STDOUT_FILENO
+#define STDOUT_FILENO 1
+#endif
+#endif
+
 namespace {
 
 class CommandLineInterfaceTest : public testing::Test {
@@ -110,6 +129,9 @@ class CommandLineInterfaceTest : public testing::Test {
                        const string& message_name,
                        const string& output_file);
 
+  void ReadDescriptorSet(const string& filename,
+                         FileDescriptorSet* descriptor_set);
+
  private:
   // The object we are testing.
   CommandLineInterface cli_;
@@ -333,6 +355,18 @@ void CommandLineInterfaceTest::ExpectGenerated(
     << "Output file did not have expected contents: " + output_file;
 }
 
+void CommandLineInterfaceTest::ReadDescriptorSet(
+    const string& filename, FileDescriptorSet* descriptor_set) {
+  string path = temp_directory_ + "/" + filename;
+  string file_contents;
+  if (!File::ReadFileToString(path, &file_contents)) {
+    FAIL() << "File not found: " << path;
+  }
+  if (!descriptor_set->ParseFromString(file_contents)) {
+    FAIL() << "Could not parse file contents: " << path;
+  }
+}
+
 // ===================================================================
 
 CommandLineInterfaceTest::MockCodeGenerator::MockCodeGenerator(
@@ -665,6 +699,57 @@ TEST_F(CommandLineInterfaceTest, CwdRelativeInputs) {
   ExpectGenerated("test_generator", "", "foo.proto", "Foo", "output.test");
 }
 
+TEST_F(CommandLineInterfaceTest, WriteDescriptorSet) {
+  CreateTempFile("foo.proto",
+    "syntax = \"proto2\";\n"
+    "message Foo {}\n");
+  CreateTempFile("bar.proto",
+    "syntax = \"proto2\";\n"
+    "import \"foo.proto\";\n"
+    "message Bar {\n"
+    "  optional Foo foo = 1;\n"
+    "}\n");
+
+  Run("protocol_compiler --descriptor_set_out=$tmpdir/descriptor_set "
+      "--proto_path=$tmpdir bar.proto");
+
+  ExpectNoErrors();
+
+  FileDescriptorSet descriptor_set;
+  ReadDescriptorSet("descriptor_set", &descriptor_set);
+  if (HasFatalFailure()) return;
+  ASSERT_EQ(1, descriptor_set.file_size());
+  EXPECT_EQ("bar.proto", descriptor_set.file(0).name());
+}
+
+TEST_F(CommandLineInterfaceTest, WriteTransitiveDescriptorSet) {
+  CreateTempFile("foo.proto",
+    "syntax = \"proto2\";\n"
+    "message Foo {}\n");
+  CreateTempFile("bar.proto",
+    "syntax = \"proto2\";\n"
+    "import \"foo.proto\";\n"
+    "message Bar {\n"
+    "  optional Foo foo = 1;\n"
+    "}\n");
+
+  Run("protocol_compiler --descriptor_set_out=$tmpdir/descriptor_set "
+      "--include_imports --proto_path=$tmpdir bar.proto");
+
+  ExpectNoErrors();
+
+  FileDescriptorSet descriptor_set;
+  ReadDescriptorSet("descriptor_set", &descriptor_set);
+  if (HasFatalFailure()) return;
+  ASSERT_EQ(2, descriptor_set.file_size());
+  if (descriptor_set.file(0).name() == "bar.proto") {
+    swap(descriptor_set.mutable_file()->mutable_data()[0],
+         descriptor_set.mutable_file()->mutable_data()[1]);
+  }
+  EXPECT_EQ("foo.proto", descriptor_set.file(0).name());
+  EXPECT_EQ("bar.proto", descriptor_set.file(1).name());
+}
+
 // -------------------------------------------------------------------
 
 TEST_F(CommandLineInterfaceTest, ParseErrors) {
@@ -954,14 +1039,14 @@ TEST_F(CommandLineInterfaceTest, HelpText) {
 TEST_F(CommandLineInterfaceTest, ParseSingleCharacterFlag) {
   // Test that a single-character flag works.
 
-  RegisterGenerator("test_generator", "-o",
+  RegisterGenerator("test_generator", "-t",
                     "output.test", "Test output.");
 
   CreateTempFile("foo.proto",
     "syntax = \"proto2\";\n"
     "message Foo {}\n");
 
-  Run("protocol_compiler -o$tmpdir "
+  Run("protocol_compiler -t$tmpdir "
       "--proto_path=$tmpdir foo.proto");
 
   ExpectNoErrors();
@@ -989,14 +1074,14 @@ TEST_F(CommandLineInterfaceTest, ParseSingleCharacterSpaceDelimitedValue) {
   // Test that separating the flag value with a space works for
   // single-character flags.
 
-  RegisterGenerator("test_generator", "-o",
+  RegisterGenerator("test_generator", "-t",
                     "output.test", "Test output.");
 
   CreateTempFile("foo.proto",
     "syntax = \"proto2\";\n"
     "message Foo {}\n");
 
-  Run("protocol_compiler -o $tmpdir "
+  Run("protocol_compiler -t $tmpdir "
       "--proto_path=$tmpdir foo.proto");
 
   ExpectNoErrors();
@@ -1026,6 +1111,166 @@ TEST_F(CommandLineInterfaceTest, MissingValueAtEndError) {
   ExpectErrorText("Missing value for flag: --test_out\n");
 }
 
+// ===================================================================
+
+// Test for --encode and --decode.  Note that it would be easier to do this
+// test as a shell script, but we'd like to be able to run the test on
+// platforms that don't have a Bourne-compatible shell available (especially
+// Windows/MSVC).
+class EncodeDecodeTest : public testing::Test {
+ protected:
+  virtual void SetUp() {
+    duped_stdin_ = dup(STDIN_FILENO);
+  }
+
+  virtual void TearDown() {
+    dup2(duped_stdin_, STDIN_FILENO);
+    close(duped_stdin_);
+  }
+
+  void RedirectStdinFromText(const string& input) {
+    string filename = TestTempDir() + "/test_stdin";
+    File::WriteStringToFileOrDie(input, filename);
+    GOOGLE_CHECK(RedirectStdinFromFile(filename));
+  }
+
+  bool RedirectStdinFromFile(const string& filename) {
+    int fd = open(filename.c_str(), O_RDONLY);
+    if (fd < 0) return false;
+    dup2(fd, STDIN_FILENO);
+    close(fd);
+    return true;
+  }
+
+  // Remove '\r' characters from text.
+  string StripCR(const string& text) {
+    string result;
+
+    for (int i = 0; i < text.size(); i++) {
+      if (text[i] != '\r') {
+        result.push_back(text[i]);
+      }
+    }
+
+    return result;
+  }
+
+  enum Type { TEXT, BINARY };
+  enum ReturnCode { SUCCESS, ERROR };
+
+  bool Run(const string& command) {
+    vector<string> args;
+    args.push_back("protoc");
+    SplitStringUsing(command, " ", &args);
+    args.push_back("--proto_path=" + TestSourceDir());
+
+    scoped_array<const char*> argv(new const char*[args.size()]);
+    for (int i = 0; i < args.size(); i++) {
+      argv[i] = args[i].c_str();
+    }
+
+    CommandLineInterface cli;
+    cli.SetInputsAreProtoPathRelative(true);
+
+    CaptureTestStdout();
+    CaptureTestStderr();
+
+    int result = cli.Run(args.size(), argv.get());
+
+    captured_stdout_ = GetCapturedTestStdout();
+    captured_stderr_ = GetCapturedTestStderr();
+
+    return result == 0;
+  }
+
+  void ExpectStdoutMatchesBinaryFile(const string& filename) {
+    string expected_output;
+    ASSERT_TRUE(File::ReadFileToString(filename, &expected_output));
+
+    // Don't use EXPECT_EQ because we don't want to print raw binary data to
+    // stdout on failure.
+    EXPECT_TRUE(captured_stdout_ == expected_output);
+  }
+
+  void ExpectStdoutMatchesTextFile(const string& filename) {
+    string expected_output;
+    ASSERT_TRUE(File::ReadFileToString(filename, &expected_output));
+
+    ExpectStdoutMatchesText(expected_output);
+  }
+
+  void ExpectStdoutMatchesText(const string& expected_text) {
+    EXPECT_EQ(StripCR(expected_text), StripCR(captured_stdout_));
+  }
+
+  void ExpectStderrMatchesText(const string& expected_text) {
+    EXPECT_EQ(StripCR(expected_text), StripCR(captured_stderr_));
+  }
+
+ private:
+  int duped_stdin_;
+  string captured_stdout_;
+  string captured_stderr_;
+};
+
+TEST_F(EncodeDecodeTest, Encode) {
+  RedirectStdinFromFile(TestSourceDir() +
+    "/google/protobuf/testdata/text_format_unittest_data.txt");
+  EXPECT_TRUE(Run("google/protobuf/unittest.proto "
+                  "--encode=protobuf_unittest.TestAllTypes"));
+  ExpectStdoutMatchesBinaryFile(TestSourceDir() +
+    "/google/protobuf/testdata/golden_message");
+  ExpectStderrMatchesText("");
+}
+
+TEST_F(EncodeDecodeTest, Decode) {
+  RedirectStdinFromFile(TestSourceDir() +
+    "/google/protobuf/testdata/golden_message");
+  EXPECT_TRUE(Run("google/protobuf/unittest.proto "
+                  "--decode=protobuf_unittest.TestAllTypes"));
+  ExpectStdoutMatchesTextFile(TestSourceDir() +
+    "/google/protobuf/testdata/text_format_unittest_data.txt");
+  ExpectStderrMatchesText("");
+}
+
+TEST_F(EncodeDecodeTest, Partial) {
+  RedirectStdinFromText("");
+  EXPECT_TRUE(Run("google/protobuf/unittest.proto "
+                  "--encode=protobuf_unittest.TestRequired"));
+  ExpectStdoutMatchesText("");
+  ExpectStderrMatchesText(
+    "warning:  Input message is missing required fields:  a, b, c\n");
+}
+
+TEST_F(EncodeDecodeTest, DecodeRaw) {
+  protobuf_unittest::TestAllTypes message;
+  message.set_optional_int32(123);
+  message.set_optional_string("foo");
+  string data;
+  message.SerializeToString(&data);
+
+  RedirectStdinFromText(data);
+  EXPECT_TRUE(Run("--decode_raw"));
+  ExpectStdoutMatchesText("1: 123\n"
+                          "14: \"foo\"\n");
+  ExpectStderrMatchesText("");
+}
+
+TEST_F(EncodeDecodeTest, UnknownType) {
+  EXPECT_FALSE(Run("google/protobuf/unittest.proto "
+                   "--encode=NoSuchType"));
+  ExpectStdoutMatchesText("");
+  ExpectStderrMatchesText("Type not defined: NoSuchType\n");
+}
+
+TEST_F(EncodeDecodeTest, ProtoParseError) {
+  EXPECT_FALSE(Run("google/protobuf/no_such_file.proto "
+                   "--encode=NoSuchType"));
+  ExpectStdoutMatchesText("");
+  ExpectStderrMatchesText(
+    "google/protobuf/no_such_file.proto: File not found.\n");
+}
+
 }  // anonymous namespace
 
 }  // namespace compiler

src/google/protobuf/compiler/cpp/cpp_file.cc

@@ -128,7 +128,14 @@ void FileGenerator::GenerateHeader(io::Printer* printer) {
   // Open namespace.
   GenerateNamespaceOpeners(printer);
 
-  printer->Print("\n");
+  // Forward-declare the BuildDescriptors function, so that we can declare it
+  // to be a friend of each class.
+  printer->Print(
+    "\n"
+    "// Internal implementation detail -- do not call this.\n"
+    "void $builddescriptorsname$();\n"
+    "\n",
+    "builddescriptorsname", GlobalBuildDescriptorsName(file_->name()));
 
   // Generate forward declarations of classes.
   for (int i = 0; i < file_->message_type_count(); i++) {
@@ -302,6 +309,9 @@ void FileGenerator::GenerateBuildDescriptors(io::Printer* printer) {
   // time, because every message has a statically-initialized default instance,
   // and the constructor for a message class accesses its descriptor.  See the
   // constructor and the descriptor() method of message classes.
+  //
+  // We also construct the reflection object for each class inside
+  // BuildDescriptors().
   printer->Print(
     "\n"
     "void $builddescriptorsname$() {\n"

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -188,7 +188,7 @@ string FilenameIdentifier(const string& filename) {
 
 // Return the name of the BuildDescriptors() function for a given file.
 string GlobalBuildDescriptorsName(const string& filename) {
-  return "proto_BuildDescriptors_" + FilenameIdentifier(filename);
+  return "protobuf_BuildDesc_" + FilenameIdentifier(filename);
 }
 
 }  // namespace cpp

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -374,6 +374,8 @@ GenerateClassDefinition(io::Printer* printer) {
   } else {
     vars["dllexport"] = dllexport_decl_ + " ";
   }
+  vars["builddescriptorsname"] =
+    GlobalBuildDescriptorsName(descriptor_->file()->name());
 
   printer->Print(vars,
     "class $dllexport$$classname$ : public ::google::protobuf::Message {\n"
@@ -396,11 +398,11 @@ GenerateClassDefinition(io::Printer* printer) {
     "}\n"
     "\n"
     "inline const ::google::protobuf::UnknownFieldSet& unknown_fields() const {\n"
-    "  return _reflection_.unknown_fields();\n"
+    "  return _unknown_fields_;\n"
     "}\n"
     "\n"
     "inline ::google::protobuf::UnknownFieldSet* mutable_unknown_fields() {\n"
-    "  return _reflection_.mutable_unknown_fields();\n"
+    "  return &_unknown_fields_;\n"
     "}\n"
     "\n"
     "static const ::google::protobuf::Descriptor* descriptor();\n"
@@ -432,8 +434,7 @@ GenerateClassDefinition(io::Printer* printer) {
     "public:\n"
     "\n"
     "const ::google::protobuf::Descriptor* GetDescriptor() const;\n"
-    "const ::google::protobuf::Message::Reflection* GetReflection() const;\n"
-    "::google::protobuf::Message::Reflection* GetReflection();\n"
+    "const ::google::protobuf::Reflection* GetReflection() const;\n"
     "\n"
     "// nested types ----------------------------------------------------\n"
     "\n");
@@ -481,7 +482,7 @@ GenerateClassDefinition(io::Printer* printer) {
 
   // TODO(kenton):  Make _cached_size_ an atomic<int> when C++ supports it.
   printer->Print(
-    "::google::protobuf::internal::GeneratedMessageReflection _reflection_;\n"
+    "::google::protobuf::UnknownFieldSet _unknown_fields_;\n"
     "mutable int _cached_size_;\n"
     "\n");
   for (int i = 0; i < descriptor_->field_count(); i++) {
@@ -491,7 +492,7 @@ GenerateClassDefinition(io::Printer* printer) {
 
   // Generate offsets and _has_bits_ boilerplate.
   printer->Print(vars,
-    "\n"
+    "friend void $builddescriptorsname$();\n"
     "static const $classname$ default_instance_;\n");
 
   if (descriptor_->field_count() > 0) {
@@ -540,8 +541,11 @@ GenerateInlineMethods(io::Printer* printer) {
 
 void MessageGenerator::
 GenerateDescriptorDeclarations(io::Printer* printer) {
-  printer->Print("const ::google::protobuf::Descriptor* $name$_descriptor_ = NULL;\n",
-                 "name", classname_);
+  printer->Print(
+    "const ::google::protobuf::Descriptor* $name$_descriptor_ = NULL;\n"
+    "const ::google::protobuf::internal::GeneratedMessageReflection*\n"
+    "  $name$_reflection_ = NULL;\n",
+    "name", classname_);
 
   for (int i = 0; i < descriptor_->nested_type_count(); i++) {
     nested_generators_[i]->GenerateDescriptorDeclarations(printer);
@@ -562,6 +566,7 @@ GenerateDescriptorInitializer(io::Printer* printer, int index) {
   vars["classname"] = classname_;
   vars["index"] = SimpleItoa(index);
 
+  // Obtain the descriptor from the parent's descriptor.
   if (descriptor_->containing_type() == NULL) {
     printer->Print(vars,
       "$classname$_descriptor_ = file->message_type($index$);\n");
@@ -572,6 +577,29 @@ GenerateDescriptorInitializer(io::Printer* printer, int index) {
         "$parent$_descriptor_->nested_type($index$);\n");
   }
 
+  // Construct the reflection object.
+  printer->Print(vars,
+    "$classname$_reflection_ =\n"
+    "  new ::google::protobuf::internal::GeneratedMessageReflection(\n"
+    "    $classname$_descriptor_,\n"
+    "    &$classname$::default_instance(),\n"
+    "    $classname$::_offsets_,\n"
+    "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET($classname$, _has_bits_[0]),\n"
+    "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET("
+      "$classname$, _unknown_fields_),\n");
+  if (descriptor_->extension_range_count() > 0) {
+    printer->Print(vars,
+      "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET("
+        "$classname$, _extensions_),\n");
+  } else {
+    // No extensions.
+    printer->Print(vars,
+      "    -1,\n");
+  }
+  printer->Print(vars,
+    "    ::google::protobuf::DescriptorPool::generated_pool());\n");
+
+  // Handle nested types.
   for (int i = 0; i < descriptor_->nested_type_count(); i++) {
     nested_generators_[i]->GenerateDescriptorInitializer(printer, i);
   }
@@ -650,15 +678,13 @@ GenerateClassMethods(io::Printer* printer) {
     "  return descriptor();\n"
     "}\n"
     "\n"
-    "const ::google::protobuf::Message::Reflection*\n"
-    "$classname$::GetReflection() const {\n"
-    "  return &_reflection_;\n"
-    "}\n"
-    "\n"
-    "::google::protobuf::Message::Reflection* $classname$::GetReflection() {\n"
-    "  return &_reflection_;\n"
+    "const ::google::protobuf::Reflection* $classname$::GetReflection() const {\n"
+    "  if ($classname$_reflection_ == NULL) $builddescriptorsname$();\n"
+    "  return $classname$_reflection_;\n"
     "}\n",
-    "classname", classname_);
+    "classname", classname_,
+    "builddescriptorsname",
+      GlobalBuildDescriptorsName(descriptor_->file()->name()));
 }
 
 void MessageGenerator::
@@ -686,20 +712,16 @@ GenerateInitializerList(io::Printer* printer) {
   printer->Indent();
   printer->Indent();
 
-  bool has_extensions = descriptor_->extension_range_count() > 0;
-  if (has_extensions) {
+  if (descriptor_->extension_range_count() > 0) {
     printer->Print(
-      "_extensions_(descriptor(),\n"
+      "_extensions_(&$classname$_descriptor_,\n"
       "             ::google::protobuf::DescriptorPool::generated_pool(),\n"
-      "             ::google::protobuf::MessageFactory::generated_factory()),\n");
+      "             ::google::protobuf::MessageFactory::generated_factory()),\n",
+      "classname", classname_);
   }
 
   printer->Print(
-    "_reflection_(descriptor(),\n"
-    "             this, &default_instance_,\n"
-    "             _offsets_, _has_bits_, $extensions$),\n"
-    "_cached_size_(0)",
-    "extensions", has_extensions ? "&_extensions_" : "NULL");
+    "_cached_size_(0)");
 
   // Write the initializers for each field.
   for (int i = 0; i < descriptor_->field_count(); i++) {
@@ -904,8 +926,7 @@ GenerateMergeFrom(io::Printer* printer) {
       "  ::google::protobuf::internal::dynamic_cast_if_available<const $classname$*>(\n"
       "    &from);\n"
       "if (source == NULL) {\n"
-      "  ::google::protobuf::internal::ReflectionOps::Merge(\n"
-      "    descriptor(), *from.GetReflection(), &_reflection_);\n"
+      "  ::google::protobuf::internal::ReflectionOps::Merge(from, this);\n"
       "} else {\n"
       "  MergeFrom(*source);\n"
       "}\n",
@@ -1028,7 +1049,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
       "bool $classname$::MergePartialFromCodedStream(\n"
       "    ::google::protobuf::io::CodedInputStream* input) {\n"
       "  return ::google::protobuf::internal::WireFormat::ParseAndMergePartial(\n"
-      "    descriptor(), input, &_reflection_);\n"
+      "    input, this);\n"
       "}\n",
       "classname", classname_);
     return;
@@ -1157,7 +1178,7 @@ GenerateMergeFromCodedStream(io::Printer* printer) {
       }
     }
     printer->Print(") {\n"
-      "  DO_(_extensions_.ParseField(tag, input, &_reflection_));\n"
+      "  DO_(_extensions_.ParseField(tag, input, this));\n"
       "  continue;\n"
       "}\n");
   }
@@ -1214,7 +1235,7 @@ void MessageGenerator::GenerateSerializeOneExtensionRange(
   printer->Print(vars,
     "// Extension range [$start$, $end$)\n"
     "DO_(_extensions_.SerializeWithCachedSizes(\n"
-    "    $start$, $end$, &_reflection_, output));\n\n");
+    "    $start$, $end$, *this, output));\n\n");
 }
 
 void MessageGenerator::
@@ -1341,7 +1362,7 @@ GenerateByteSize(io::Printer* printer) {
 
   if (descriptor_->extension_range_count() > 0) {
     printer->Print(
-      "total_size += _extensions_.ByteSize(&_reflection_);\n"
+      "total_size += _extensions_.ByteSize(*this);\n"
       "\n");
   }
 

src/google/protobuf/compiler/cpp/cpp_unittest.cc

@@ -268,7 +268,7 @@ TEST(GeneratedMessageTest, DynamicMessageCopyFrom) {
 
   TestUtil::ReflectionTester reflection_tester(
     unittest::TestAllTypes::descriptor());
-  reflection_tester.SetAllFieldsViaReflection(message1->GetReflection());
+  reflection_tester.SetAllFieldsViaReflection(message1.get());
 
   message2.CopyFrom(*message1);
 

src/google/protobuf/compiler/parser.cc

@@ -604,7 +604,7 @@ bool Parser::ParseDefaultAssignment(FieldDescriptorProto* field) {
 }
 
 bool Parser::ParseOptionAssignment(Message* options) {
-  Message::Reflection* reflection = options->GetReflection();
+  const Reflection* reflection = options->GetReflection();
   const Descriptor* descriptor = options->GetDescriptor();
 
   // Parse name.
@@ -623,7 +623,7 @@ bool Parser::ParseOptionAssignment(Message* options) {
     AddError(line, column, "Not implemented: repeated options.");
     return false;
   }
-  if (reflection->HasField(field)) {
+  if (reflection->HasField(*options, field)) {
     AddError(line, column, "Option \"" + name + "\" was already set.");
     return false;
   }
@@ -638,7 +638,7 @@ bool Parser::ParseOptionAssignment(Message* options) {
 
     // This field is a message/group.  The user must identify a field within
     // it to set.
-    return ParseOptionAssignment(reflection->MutableMessage(field));
+    return ParseOptionAssignment(reflection->MutableMessage(options, field));
   }
 
   DO(Consume("="));
@@ -651,7 +651,7 @@ bool Parser::ParseOptionAssignment(Message* options) {
       uint64 max_value = kint32max;
       if (is_negative) ++max_value;
       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
-      reflection->SetInt32(field, is_negative ? -value : value);
+      reflection->SetInt32(options, field, is_negative ? -value : value);
       break;
     }
 
@@ -661,21 +661,21 @@ bool Parser::ParseOptionAssignment(Message* options) {
       uint64 max_value = kint64max;
       if (is_negative) ++max_value;
       DO(ConsumeInteger64(max_value, &value, "Expected integer."));
-      reflection->SetInt64(field, is_negative ? -value : value);
+      reflection->SetInt64(options, field, is_negative ? -value : value);
       break;
     }
 
     case FieldDescriptor::CPPTYPE_UINT32: {
       uint64 value;
       DO(ConsumeInteger64(kuint32max, &value, "Expected integer."));
-      reflection->SetUInt32(field, value);
+      reflection->SetUInt32(options, field, value);
       break;
     }
 
     case FieldDescriptor::CPPTYPE_UINT64: {
       uint64 value;
       DO(ConsumeInteger64(kuint64max, &value, "Expected integer."));
-      reflection->SetUInt64(field, value);
+      reflection->SetUInt64(options, field, value);
       break;
     }
 
@@ -683,7 +683,7 @@ bool Parser::ParseOptionAssignment(Message* options) {
       double value;
       bool is_negative = TryConsume("-");
       DO(ConsumeNumber(&value, "Expected number."));
-      reflection->SetDouble(field, is_negative ? -value : value);
+      reflection->SetDouble(options, field, is_negative ? -value : value);
       break;
     }
 
@@ -691,15 +691,15 @@ bool Parser::ParseOptionAssignment(Message* options) {
       double value;
       bool is_negative = TryConsume("-");
       DO(ConsumeNumber(&value, "Expected number."));
-      reflection->SetFloat(field, is_negative ? -value : value);
+      reflection->SetFloat(options, field, is_negative ? -value : value);
       break;
     }
 
     case FieldDescriptor::CPPTYPE_BOOL:
       if (TryConsume("true")) {
-        reflection->SetBool(field, true);
+        reflection->SetBool(options, field, true);
       } else if (TryConsume("false")) {
-        reflection->SetBool(field, false);
+        reflection->SetBool(options, field, false);
       } else {
         AddError("Expected \"true\" or \"false\".");
         return false;
@@ -719,14 +719,14 @@ bool Parser::ParseOptionAssignment(Message* options) {
           "named \"" + value_name + "\".");
         return false;
       }
-      reflection->SetEnum(field, value);
+      reflection->SetEnum(options, field, value);
       break;
     }
 
     case FieldDescriptor::CPPTYPE_STRING: {
       string value;
       DO(ConsumeString(&value, "Expected string."));
-      reflection->SetString(field, value);
+      reflection->SetString(options, field, value);
       break;
     }
 

src/google/protobuf/descriptor.cc

@@ -1232,9 +1232,9 @@ namespace {
 // Used by each of the option formatters.
 bool RetrieveOptions(const Message &options, vector<string> *option_entries) {
   option_entries->clear();
-  const Message::Reflection *reflection = options.GetReflection();
+  const Reflection* reflection = options.GetReflection();
   vector<const FieldDescriptor*> fields;
-  reflection->ListFields(&fields);
+  reflection->ListFields(options, &fields);
   for (int i = 0; i < fields.size(); i++) {
     // Doesn't make sense to have message type fields here
     if (fields[i]->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
@@ -1243,7 +1243,7 @@ bool RetrieveOptions(const Message &options, vector<string> *option_entries) {
     int count = 1;
     bool repeated = false;
     if (fields[i]->is_repeated()) {
-      count = reflection->FieldSize(fields[i]);
+      count = reflection->FieldSize(options, fields[i]);
       repeated = true;
     }
     for (int j = 0; j < count; j++) {

src/google/protobuf/descriptor.h

@@ -232,9 +232,9 @@ class LIBPROTOBUF_EXPORT Descriptor {
 //   call Descriptor::FindExtensionByName() or
 //   FileDescriptor::FindExtensionByName().
 // - Given a DescriptorPool, call DescriptorPool::FindExtensionByNumber().
-// - Given a Message::Reflection for a message object, call
-//   Message::Reflection::FindKnownExtensionByName() or
-//   Message::Reflection::FindKnownExtensionByNumber().
+// - Given a Reflection for a message object, call
+//   Reflection::FindKnownExtensionByName() or
+//   Reflection::FindKnownExtensionByNumber().
 // Use DescriptorPool to construct your own descriptors.
 class LIBPROTOBUF_EXPORT FieldDescriptor {
  public:

src/google/protobuf/descriptor.proto

@@ -32,6 +32,12 @@ option java_outer_classname = "DescriptorProtos";
 // algorithms don't work during bootstrapping.
 option optimize_for = SPEED;
 
+// The protocol compiler can output a FileDescriptorSet containing the .proto
+// files it parses.
+message FileDescriptorSet {
+  repeated FileDescriptorProto file = 1;
+}
+
 // Describes a complete .proto file.
 message FileDescriptorProto {
   optional string name = 1;       // file name, relative to root of source tree

src/google/protobuf/dynamic_message_unittest.cc

@@ -90,7 +90,7 @@ TEST_F(DynamicMessageTest, OnePrototype) {
 TEST_F(DynamicMessageTest, Defaults) {
   // Check that all default values are set correctly in the initial message.
   TestUtil::ReflectionTester reflection_tester(descriptor_);
-  reflection_tester.ExpectClearViaReflection(*prototype_->GetReflection());
+  reflection_tester.ExpectClearViaReflection(*prototype_);
 }
 
 TEST_F(DynamicMessageTest, IndependentOffsets) {
@@ -100,8 +100,8 @@ TEST_F(DynamicMessageTest, IndependentOffsets) {
   scoped_ptr<Message> message(prototype_->New());
   TestUtil::ReflectionTester reflection_tester(descriptor_);
 
-  reflection_tester.SetAllFieldsViaReflection(message->GetReflection());
-  reflection_tester.ExpectAllFieldsSetViaReflection(*message->GetReflection());
+  reflection_tester.SetAllFieldsViaReflection(message.get());
+  reflection_tester.ExpectAllFieldsSetViaReflection(*message);
 }
 
 TEST_F(DynamicMessageTest, Extensions) {
@@ -109,8 +109,8 @@ TEST_F(DynamicMessageTest, Extensions) {
   scoped_ptr<Message> message(extensions_prototype_->New());
   TestUtil::ReflectionTester reflection_tester(extensions_descriptor_);
 
-  reflection_tester.SetAllFieldsViaReflection(message->GetReflection());
-  reflection_tester.ExpectAllFieldsSetViaReflection(*message->GetReflection());
+  reflection_tester.SetAllFieldsViaReflection(message.get());
+  reflection_tester.ExpectAllFieldsSetViaReflection(*message);
 }
 
 }  // namespace protobuf

src/google/protobuf/extension_set.cc

@@ -34,42 +34,10 @@ namespace internal {
 // -------------------------------------------------------------------
 // Lookup functions
 
-const FieldDescriptor*
-ExtensionSet::FindKnownExtensionByName(const string& name) const {
-  const FieldDescriptor* result = descriptor_pool_->FindExtensionByName(name);
-  if (result != NULL && result->containing_type() == extendee_) {
-    return result;
-  }
-
-  if (extendee_->options().message_set_wire_format()) {
-    // MessageSet extensions may be identified by type name.
-    const Descriptor* type = descriptor_pool_->FindMessageTypeByName(name);
-    if (type != NULL) {
-      // Look for a matching extension in the foreign type's scope.
-      for (int i = 0; i < type->extension_count(); i++) {
-        const FieldDescriptor* extension = type->extension(i);
-        if (extension->containing_type() == extendee_ &&
-            extension->type() == FieldDescriptor::TYPE_MESSAGE &&
-            extension->is_optional() &&
-            extension->message_type() == type) {
-          // Found it.
-          return extension;
-        }
-      }
-    }
-  }
-
-  return NULL;
-}
-
-const FieldDescriptor*
-ExtensionSet::FindKnownExtensionByNumber(int number) const {
-  return descriptor_pool_->FindExtensionByNumber(extendee_, number);
-}
-
 const FieldDescriptor*
 ExtensionSet::FindKnownExtensionOrDie(int number) const {
-  const FieldDescriptor* descriptor = FindKnownExtensionByNumber(number);
+  const FieldDescriptor* descriptor =
+    descriptor_pool_->FindExtensionByNumber(*extendee_, number);
   if (descriptor == NULL) {
     // This extension doesn't exist, so we have to crash.  However, let's
     // try to provide an informative error message.
@@ -77,7 +45,7 @@ ExtensionSet::FindKnownExtensionOrDie(int number) const {
         message_factory_ == MessageFactory::generated_factory()) {
       // This is probably the ExtensionSet for a generated class.
       GOOGLE_LOG(FATAL) << ": No extension is registered for \""
-                 << extendee_->full_name() << "\" with number "
+                 << (*extendee_)->full_name() << "\" with number "
                  << number << ".  Perhaps you were trying to access it via "
                     "the Reflection interface, but you provided a "
                     "FieldDescriptor which did not come from a linked-in "
@@ -87,7 +55,7 @@ ExtensionSet::FindKnownExtensionOrDie(int number) const {
     } else {
       // This is probably a DynamicMessage.
       GOOGLE_LOG(FATAL) << ": No extension is registered for \""
-                 << extendee_->full_name() << "\" with number "
+                 << (*extendee_)->full_name() << "\" with number "
                  << number << ".  If you were using a DynamicMessage, "
                     "remember that you are only allowed to access extensions "
                     "which are defined in the DescriptorPool which you passed "
@@ -105,7 +73,7 @@ ExtensionSet::GetPrototype(const Descriptor* message_type) const {
 // ===================================================================
 // Constructors and basic methods.
 
-ExtensionSet::ExtensionSet(const Descriptor* extendee,
+ExtensionSet::ExtensionSet(const Descriptor* const* extendee,
                            const DescriptorPool* pool,
                            MessageFactory* factory)
   : extendee_(extendee),
@@ -461,7 +429,7 @@ void MergeRepeatedFields(const RepeatedPtrField<Message>& source,
 }  // namespace
 
 void ExtensionSet::MergeFrom(const ExtensionSet& other) {
-  GOOGLE_DCHECK_EQ(extendee_, other.extendee_);
+  GOOGLE_DCHECK_EQ(*extendee_, *other.extendee_);
 
   for (map<int, Extension>::const_iterator iter = other.extensions_.begin();
        iter != other.extensions_.end(); ++iter) {
@@ -558,22 +526,23 @@ bool ExtensionSet::IsInitialized() const {
 }
 
 bool ExtensionSet::ParseField(uint32 tag, io::CodedInputStream* input,
-                              Message::Reflection* reflection) {
+                              Message* message) {
   const FieldDescriptor* field =
-    FindKnownExtensionByNumber(WireFormat::GetTagFieldNumber(tag));
+    message->GetReflection()
+           ->FindKnownExtensionByNumber(WireFormat::GetTagFieldNumber(tag));
 
-  return WireFormat::ParseAndMergeField(tag, field, reflection, input);
+  return WireFormat::ParseAndMergeField(tag, field, message, input);
 }
 
 bool ExtensionSet::SerializeWithCachedSizes(
     int start_field_number, int end_field_number,
-    const Message::Reflection* reflection,
+    const Message& message,
     io::CodedOutputStream* output) const {
   map<int, Extension>::const_iterator iter;
   for (iter = extensions_.lower_bound(start_field_number);
        iter != extensions_.end() && iter->first < end_field_number;
        ++iter) {
-    if (!iter->second.SerializeFieldWithCachedSizes(reflection, output)) {
+    if (!iter->second.SerializeFieldWithCachedSizes(message, output)) {
       return false;
     }
   }
@@ -581,12 +550,12 @@ bool ExtensionSet::SerializeWithCachedSizes(
   return true;
 }
 
-int ExtensionSet::ByteSize(const Message::Reflection* reflection) const {
+int ExtensionSet::ByteSize(const Message& message) const {
   int total_size = 0;
 
   for (map<int, Extension>::const_iterator iter = extensions_.begin();
        iter != extensions_.end(); ++iter) {
-    total_size += iter->second.ByteSize(reflection);
+    total_size += iter->second.ByteSize(message);
   }
 
   return total_size;
@@ -652,20 +621,19 @@ void ExtensionSet::Extension::Clear() {
 }
 
 bool ExtensionSet::Extension::SerializeFieldWithCachedSizes(
-    const Message::Reflection* reflection,
+    const Message& message,
     io::CodedOutputStream* output) const {
   if (descriptor->is_repeated() || !is_cleared) {
     return WireFormat::SerializeFieldWithCachedSizes(
-        descriptor, reflection, output);
+      descriptor, message, output);
   } else {
     return true;
   }
 }
 
-int64 ExtensionSet::Extension::ByteSize(
-    const Message::Reflection* reflection) const {
+int64 ExtensionSet::Extension::ByteSize(const Message& message) const {
   if (descriptor->is_repeated() || !is_cleared) {
-    return WireFormat::FieldByteSize(descriptor, reflection);
+    return WireFormat::FieldByteSize(descriptor, message);
   } else {
     // Cleared, non-repeated field.
     return 0;

src/google/protobuf/extension_set.h

@@ -64,26 +64,28 @@ namespace internal {
 class LIBPROTOBUF_EXPORT ExtensionSet {
  public:
   // Construct an ExtensionSet.
-  //   extendee:  Descriptor for the type being extended.
+  //   extendee:  Descriptor for the type being extended. We pass in a pointer
+  //              to a pointer to the extendee to get around an initialization
+  //              problem: when we create the ExtensionSet for a message type,
+  //              its descriptor may not exist yet. But we know where that
+  //              descriptor pointer will be placed, and by the time it's used
+  //              by this ExtensionSet it will be fully initialized, so passing
+  //              a pointer to that location works. Note that this problem
+  //              will only occur for messages defined in descriptor.proto.
   //   pool:      DescriptorPool to search for extension definitions.
   //   factory:   MessageFactory used to construct implementations of messages
   //              for extensions with message type.  This factory must be able
   //              to construct any message type found in "pool".
   // All three objects remain property of the caller and must outlive the
   // ExtensionSet.
-  ExtensionSet(const Descriptor* extendee,
+  ExtensionSet(const Descriptor* const* extendee,
                const DescriptorPool* pool,
                MessageFactory* factory);
 
   ~ExtensionSet();
 
-  // Search for a known (compiled-in) extension of this type by name or number.
-  // Returns NULL if no extension is known.
-  const FieldDescriptor* FindKnownExtensionByName(const string& name) const;
-  const FieldDescriptor* FindKnownExtensionByNumber(int number) const;
-
   // Add all fields which are currently present to the given vector.  This
-  // is useful to implement Message::Reflection::ListFields().
+  // is useful to implement Reflection::ListFields().
   void AppendToList(vector<const FieldDescriptor*>* output) const;
 
   // =================================================================
@@ -110,7 +112,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
   // die on an assert failure.  The message objects returned by the message
   // accessors are guaranteed to be of the correct linked-in type.
   //
-  // These methods pretty much match Message::Reflection except that:
+  // These methods pretty much match Reflection except that:
   // - They're not virtual.
   // - They identify fields by number rather than FieldDescriptors.
   // - They identify enum values using integers rather than descriptors.
@@ -196,7 +198,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
   bool IsInitialized() const;
 
   // These parsing and serialization functions all want a pointer to the
-  // reflection interface because they hand off the actual work to WireFormat,
+  // message object because they hand off the actual work to WireFormat,
   // which works in terms of a reflection interface.  Yes, this means there
   // are some redundant virtual function calls that end up being made, but
   // it probably doesn't matter much in practice, and the alternative would
@@ -204,8 +206,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
 
   // Parses a single extension from the input.  The input should start out
   // positioned immediately after the tag.
-  bool ParseField(uint32 tag, io::CodedInputStream* input,
-                  Message::Reflection* reflection);
+  bool ParseField(uint32 tag, io::CodedInputStream* input, Message* message);
 
   // Write all extension fields with field numbers in the range
   //   [start_field_number, end_field_number)
@@ -213,11 +214,11 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
   // last called.  Note that the range bounds are inclusive-exclusive.
   bool SerializeWithCachedSizes(int start_field_number,
                                 int end_field_number,
-                                const Message::Reflection* reflection,
+                                const Message& message,
                                 io::CodedOutputStream* output) const;
 
   // Returns the total serialized size of all the extensions.
-  int ByteSize(const Message::Reflection* reflection) const;
+  int ByteSize(const Message& message) const;
 
  private:
   // Like FindKnownExtension(), but GOOGLE_CHECK-fail if not found.
@@ -265,9 +266,9 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
 
     // Some helper methods for operations on a single Extension.
     bool SerializeFieldWithCachedSizes(
-        const Message::Reflection* reflection,
+        const Message& message,
         io::CodedOutputStream* output) const;
-    int64 ByteSize(const Message::Reflection* reflection) const;
+    int64 ByteSize(const Message& message) const;
     void Clear();
     int GetSize() const;
     void Free();
@@ -280,7 +281,7 @@ class LIBPROTOBUF_EXPORT ExtensionSet {
   // for 100 elements or more.  Also, we want AppendToList() to order fields
   // by field number.
   map<int, Extension> extensions_;
-  const Descriptor* extendee_;
+  const Descriptor* const* extendee_;
   const DescriptorPool* descriptor_pool_;
   MessageFactory* message_factory_;
 

src/google/protobuf/message.cc

@@ -50,7 +50,6 @@ static string InitializationErrorMessage(const char* action,
 }
 
 Message::~Message() {}
-Message::Reflection::~Reflection() {}
 
 void Message::MergeFrom(const Message& from) {
   const Descriptor* descriptor = GetDescriptor();
@@ -58,7 +57,7 @@ void Message::MergeFrom(const Message& from) {
     << ": Tried to merge from a message with a different type.  "
        "to: " << descriptor->full_name() << ", "
        "from:" << from.GetDescriptor()->full_name();
-  ReflectionOps::Merge(descriptor, *from.GetReflection(), GetReflection());
+  ReflectionOps::Merge(from, this);
 }
 
 void Message::CopyFrom(const Message& from) {
@@ -67,20 +66,19 @@ void Message::CopyFrom(const Message& from) {
     << ": Tried to copy from a message with a different type."
        "to: " << descriptor->full_name() << ", "
        "from:" << from.GetDescriptor()->full_name();
-  ReflectionOps::Copy(descriptor, *from.GetReflection(), GetReflection());
+  ReflectionOps::Copy(from, this);
 }
 
 void Message::Clear() {
-  ReflectionOps::Clear(GetDescriptor(), GetReflection());
+  ReflectionOps::Clear(this);
 }
 
 bool Message::IsInitialized() const {
-  return ReflectionOps::IsInitialized(GetDescriptor(), *GetReflection());
+  return ReflectionOps::IsInitialized(*this);
 }
 
 void Message::FindInitializationErrors(vector<string>* errors) const {
-  return ReflectionOps::FindInitializationErrors(
-    GetDescriptor(), *GetReflection(), "", errors);
+  return ReflectionOps::FindInitializationErrors(*this, "", errors);
 }
 
 string Message::InitializationErrorString() const {
@@ -96,12 +94,11 @@ void Message::CheckInitialized() const {
 }
 
 void Message::DiscardUnknownFields() {
-  return ReflectionOps::DiscardUnknownFields(GetDescriptor(), GetReflection());
+  return ReflectionOps::DiscardUnknownFields(this);
 }
 
 bool Message::MergePartialFromCodedStream(io::CodedInputStream* input) {
-  return WireFormat::ParseAndMergePartial(
-    GetDescriptor(), input, GetReflection());
+  return WireFormat::ParseAndMergePartial(input, this);
 }
 
 bool Message::MergeFromCodedStream(io::CodedInputStream* input) {
@@ -178,12 +175,11 @@ bool Message::ParsePartialFromIstream(istream* input) {
 
 bool Message::SerializeWithCachedSizes(
     io::CodedOutputStream* output) const {
-  return WireFormat::SerializeWithCachedSizes(
-    GetDescriptor(), GetReflection(), GetCachedSize(), output);
+  return WireFormat::SerializeWithCachedSizes(*this, GetCachedSize(), output);
 }
 
 int Message::ByteSize() const {
-  int size = WireFormat::ByteSize(GetDescriptor(), GetReflection());
+  int size = WireFormat::ByteSize(*this);
   SetCachedSize(size);
   return size;
 }
@@ -281,6 +277,8 @@ bool Message::SerializePartialToOstream(ostream* output) const {
 }
 
 
+Reflection::~Reflection() {}
+
 // ===================================================================
 // MessageFactory
 

src/google/protobuf/reflection_ops.h

@@ -30,7 +30,7 @@ namespace google {
 namespace protobuf {
 namespace internal {
 
-// Basic operations that can be performed using Message::Reflection.
+// Basic operations that can be performed using reflection.
 // These can be used as a cheap way to implement the corresponding
 // methods of the Message interface, though they are likely to be
 // slower than implementations tailored for the specific message type.
@@ -41,24 +41,16 @@ namespace internal {
 // This class is really a namespace that contains only static methods.
 class LIBPROTOBUF_EXPORT ReflectionOps {
  public:
-  static void Copy(const Descriptor* descriptor,
-                   const Message::Reflection& from,
-                   Message::Reflection* to);
-  static void Merge(const Descriptor* descriptor,
-                    const Message::Reflection& from,
-                    Message::Reflection* to);
-  static void Clear(const Descriptor* descriptor,
-                    Message::Reflection* reflection);
-  static bool IsInitialized(const Descriptor* descriptor,
-                            const Message::Reflection& reflection);
-  static void DiscardUnknownFields(const Descriptor* descriptor,
-                                   Message::Reflection* reflection);
+  static void Copy(const Message& from, Message* to);
+  static void Merge(const Message& from, Message* to);
+  static void Clear(Message* message);
+  static bool IsInitialized(const Message& message);
+  static void DiscardUnknownFields(Message* message);
 
   // Finds all unset required fields in the message and adds their full
   // paths (e.g. "foo.bar[5].baz") to *names.  "prefix" will be attached to
   // the front of each name.
-  static void FindInitializationErrors(const Descriptor* descriptor,
-                                       const Message::Reflection& reflection,
+  static void FindInitializationErrors(const Message& message,
                                        const string& prefix,
                                        vector<string>* errors);
 

src/google/protobuf/service.h

@@ -103,7 +103,7 @@ class Message;               // message.h
 // themselves are abstract interfaces (implemented either by servers or as
 // stubs), but they subclass this base interface.  The methods of this
 // interface can be used to call the methods of the Service without knowing
-// its exact type at compile time (analogous to Message::Reflection).
+// its exact type at compile time (analogous to Reflection).
 class LIBPROTOBUF_EXPORT Service {
  public:
   inline Service() {}

src/google/protobuf/testing/googletest.h

@@ -33,10 +33,12 @@ string TestSourceDir();
 // placed.
 string TestTempDir();
 
-// Capture all text written to stderr.
+// Capture all text written to stdout or stderr.
+void CaptureTestStdout();
 void CaptureTestStderr();
 
-// Stop capturing stderr and return the text captured.
+// Stop capturing stdout or stderr and return the text captured.
+string GetCapturedTestStdout();
 string GetCapturedTestStderr();
 
 // For use with ScopedMemoryLog::GetMessages().  Inside Google the LogLevel