Fixes for Python/C++ implementation in open-source:

  * Rosy hack doesn't apply (that test should be removed
    for the open-source release).

  * Added our own copy of parameterized.py (the open-source
    version of Google Apputils doesn't contain it).

  * The C++ Descriptor object didn't implement extension_ranges.

  * Had to implement a hack around returning EncodeError, to
    work around the module-loading behavior of the test runner.

Makefile.am

@@ -197,7 +197,6 @@ javanano_EXTRA_DIST=
 python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/api_implementation.cc                      \
   python/google/protobuf/internal/api_implementation.py                      \
-  python/google/protobuf/internal/api_implementation_default_test.py         \
   python/google/protobuf/internal/containers.py                              \
   python/google/protobuf/internal/cpp_message.py                             \
   python/google/protobuf/internal/decoder.py                                 \
@@ -221,6 +220,7 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/internal/more_extensions.proto                      \
   python/google/protobuf/internal/more_extensions_dynamic.proto              \
   python/google/protobuf/internal/more_messages.proto                        \
+  python/google/protobuf/internal/_parameterized.py                          \
   python/google/protobuf/internal/proto_builder_test.py                      \
   python/google/protobuf/internal/python_message.py                          \
   python/google/protobuf/internal/reflection_test.py                         \
@@ -242,16 +242,17 @@ python_EXTRA_DIST=                                                           \
   python/google/protobuf/pyext/cpp_message.py                                \
   python/google/protobuf/pyext/descriptor.h                                  \
   python/google/protobuf/pyext/descriptor.cc                                 \
-  python/google/protobuf/pyext/descriptor_cpp2_test.py                       \
+  python/google/protobuf/pyext/descriptor_pool.h                             \
+  python/google/protobuf/pyext/descriptor_pool.cc                            \
+  python/google/protobuf/pyext/descriptor_containers.h                       \
+  python/google/protobuf/pyext/descriptor_containers.cc                      \
   python/google/protobuf/pyext/extension_dict.h                              \
   python/google/protobuf/pyext/extension_dict.cc                             \
   python/google/protobuf/pyext/message.h                                     \
   python/google/protobuf/pyext/message.cc                                    \
-  python/google/protobuf/pyext/message_factory_cpp2_test.py                  \
   python/google/protobuf/pyext/proto2_api_test.proto                         \
   python/google/protobuf/pyext/python.proto                                  \
   python/google/protobuf/pyext/python_protobuf.h                             \
-  python/google/protobuf/pyext/reflection_cpp2_generated_test.py             \
   python/google/protobuf/pyext/repeated_composite_container.h                \
   python/google/protobuf/pyext/repeated_composite_container.cc               \
   python/google/protobuf/pyext/repeated_scalar_container.h                   \

python/google/protobuf/internal/message_test.py

@@ -51,10 +51,10 @@ import sys
 import unittest
 
 from google.apputils import basetest
-from google.apputils.pybase import parameterized
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf.internal import api_implementation
+from google.protobuf.internal import _parameterized
 from google.protobuf.internal import test_util
 from google.protobuf import message
 
@@ -72,7 +72,7 @@ def IsNegInf(val):
   return isinf(val) and (val < 0)
 
 
-@parameterized.Parameters(
+@_parameterized.Parameters(
     (unittest_pb2),
     (unittest_proto3_arena_pb2))
 class MessageTest(basetest.TestCase):
@@ -982,7 +982,6 @@ class Proto2Test(basetest.TestCase):
     # This is still an incomplete proto - so serializing should fail
     self.assertRaises(message.EncodeError, unpickled_message.SerializeToString)
 
-
   # TODO(haberman): this isn't really a proto2-specific test except that this
   # message has a required field in it.  Should probably be factored out so
   # that we can test the other parts with proto3.

python/google/protobuf/internal/reflection_test.py

@@ -1803,16 +1803,6 @@ class ReflectionTest(basetest.TestCase):
     self.assertRaises(TypeError,
                       unittest_pb2.TestAllTypes().__getattribute__, 42)
 
-  @basetest.unittest.skipIf(
-      api_implementation.Type() != 'cpp' or api_implementation.Version() != 2,
-      'CPPv2-specific test')
-  def testRosyHack(self):
-    from google.protobuf.pyext import _message
-    from google3.gdata.rosy.proto import core_api2_pb2
-    from google3.gdata.rosy.proto import core_pb2
-    self.assertEqual(_message.Message, core_pb2.PageSelection.__base__)
-    self.assertEqual(_message.Message, core_api2_pb2.PageSelection.__base__)
-
 
 #  Since we had so many tests for protocol buffer equality, we broke these out
 #  into separate TestCase classes.

python/google/protobuf/internal/text_format_test.py

@@ -37,12 +37,12 @@ __author__ = 'kenton@google.com (Kenton Varda)'
 import re
 
 from google.apputils import basetest
-from google.apputils.pybase import parameterized
 
 from google.protobuf import unittest_mset_pb2
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf.internal import api_implementation
+from google.protobuf.internal import _parameterized
 from google.protobuf.internal import test_util
 from google.protobuf import text_format
 
@@ -72,7 +72,7 @@ class TextFormatBase(basetest.TestCase):
     return text
 
 
-@parameterized.Parameters(
+@_parameterized.Parameters(
     (unittest_pb2),
     (unittest_proto3_arena_pb2))
 class TextFormatTest(TextFormatBase):

python/google/protobuf/message.py

@@ -36,7 +36,6 @@
 
 __author__ = 'robinson@google.com (Will Robinson)'
 
-
 class Error(Exception): pass
 class DecodeError(Error): pass
 class EncodeError(Error): pass

python/google/protobuf/pyext/descriptor.cc

@@ -433,6 +433,20 @@ static PyObject* IsExtendable(PyBaseDescriptor *self, void *closure) {
   }
 }
 
+static PyObject* GetExtensionRanges(PyBaseDescriptor *self, void *closure) {
+  const Descriptor* descriptor = _GetDescriptor(self);
+  PyObject* range_list = PyList_New(descriptor->extension_range_count());
+
+  for (int i = 0; i < descriptor->extension_range_count(); i++) {
+    const Descriptor::ExtensionRange* range = descriptor->extension_range(i);
+    PyObject* start = PyInt_FromLong(range->start);
+    PyObject* end = PyInt_FromLong(range->end);
+    PyList_SetItem(range_list, i, PyTuple_Pack(2, start, end));
+  }
+
+  return range_list;
+}
+
 static PyObject* GetContainingType(PyBaseDescriptor *self, void *closure) {
   const Descriptor* containing_type =
       _GetDescriptor(self)->containing_type();
@@ -512,6 +526,7 @@ static PyGetSetDef Getters[] = {
   { C("nested_types_by_name"), (getter)GetNestedTypesByName, NULL, "Nested types by name", NULL},
   { C("extensions"), (getter)GetExtensions, NULL, "Extensions Sequence", NULL},
   { C("extensions_by_name"), (getter)GetExtensionsByName, NULL, "Extensions by name", NULL},
+  { C("extension_ranges"), (getter)GetExtensionRanges, NULL, "Extension ranges", NULL},
   { C("enum_types"), (getter)GetEnumsSeq, NULL, "Enum sequence", NULL},
   { C("enum_types_by_name"), (getter)GetEnumTypesByName, NULL, "Enum types by name", NULL},
   { C("enum_values_by_name"), (getter)GetEnumValuesByName, NULL, "Enum values by name", NULL},

python/google/protobuf/pyext/message.cc

@@ -1264,7 +1264,27 @@ static PyObject* SerializeToString(CMessage* self, PyObject* args) {
     if (joined == NULL) {
       return NULL;
     }
-    PyErr_Format(EncodeError_class, "Message %s is missing required fields: %s",
+
+    // TODO(haberman): this is a (hopefully temporary) hack.  The unit testing
+    // infrastructure reloads all pure-Python modules for every test, but not
+    // C++ modules (because that's generally impossible:
+    // http://bugs.python.org/issue1144263).  But if we cache EncodeError, we'll
+    // return the EncodeError from a previous load of the module, which won't
+    // match a user's attempt to catch EncodeError.  So we have to look it up
+    // again every time.
+    ScopedPyObjectPtr message_module(PyImport_ImportModule(
+        "google.protobuf.message"));
+    if (message_module.get() == NULL) {
+      return NULL;
+    }
+
+    ScopedPyObjectPtr encode_error(
+        PyObject_GetAttrString(message_module, "EncodeError"));
+    if (encode_error.get() == NULL) {
+      return NULL;
+    }
+    PyErr_Format(encode_error.get(),
+                 "Message %s is missing required fields: %s",
                  GetMessageName(self).c_str(), PyString_AsString(joined));
     return NULL;
   }
@@ -2284,8 +2304,8 @@ int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
 
 PyTypeObject CMessage_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  "google.protobuf.internal."
-  "cpp._message.CMessage",             // tp_name
+  "google.protobuf."
+  "pyext._message.CMessage",           // tp_name
   sizeof(CMessage),                    // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)cmessage::Dealloc,       //  tp_dealloc

python/google/protobuf/pyext/repeated_composite_container.cc

@@ -732,8 +732,8 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject RepeatedCompositeContainer_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  "google.protobuf.internal."
-  "cpp._message.RepeatedCompositeContainer",  // tp_name
+  "google.protobuf.pyext."
+  "_message.RepeatedCompositeContainer",  // tp_name
   sizeof(RepeatedCompositeContainer),     // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)repeated_composite_container::Dealloc,  //  tp_dealloc

python/google/protobuf/pyext/repeated_scalar_container.cc

@@ -769,8 +769,8 @@ static PyMethodDef Methods[] = {
 
 PyTypeObject RepeatedScalarContainer_Type = {
   PyVarObject_HEAD_INIT(&PyType_Type, 0)
-  "google.protobuf.internal."
-  "cpp._message.RepeatedScalarContainer",  // tp_name
+  "google.protobuf."
+  "pyext._message.RepeatedScalarContainer",  // tp_name
   sizeof(RepeatedScalarContainer),     // tp_basicsize
   0,                                   //  tp_itemsize
   (destructor)repeated_scalar_container::Dealloc,  //  tp_dealloc

src/Makefile.am

@@ -307,6 +307,7 @@ EXTRA_DIST =                                                   \
   google/protobuf/io/gzip_stream_unittest.sh                   \
   google/protobuf/testdata/golden_message                      \
   google/protobuf/testdata/golden_message_oneof_implemented    \
+  google/protobuf/testdata/golden_message_proto3               \
   google/protobuf/testdata/golden_packed_fields_message        \
   google/protobuf/testdata/bad_utf8_string                     \
   google/protobuf/testdata/text_format_unittest_data.txt       \