Merge branch 'master' of github.com:google/protobuf

csharp/Google.Protobuf.Tools.nuspec

+<?xml version="1.0" encoding="utf-8"?>
+<package>
+  <metadata>
+    <id>Google.Protobuf.Tools</id>
+    <title>Google Protocol Buffers tools</title>
+    <summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
+    <description>See project site for more info.</description>
+    <version>3.0.0-beta2</version>
+    <authors>Google Inc.</authors>
+    <owners>protobuf-packages</owners>
+    <licenseUrl>https://github.com/google/protobuf/blob/master/LICENSE</licenseUrl>
+    <projectUrl>https://github.com/google/protobuf</projectUrl>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <releaseNotes>Tools for Protocol Buffers</releaseNotes>
+    <copyright>Copyright 2015, Google Inc.</copyright>
+    <tags>Protocol Buffers Binary Serialization Format Google proto proto3</tags>
+  </metadata>
+  <files>
+    <file src="protoc\windows_x86\protoc.exe" target="tools\windows_x86\protoc.exe" />
+    <file src="protoc\windows_x64\protoc.exe" target="tools\windows_x64\protoc.exe" />
+    <file src="protoc\linux_x86\protoc" target="tools\linux_x86\protoc" />
+    <file src="protoc\linux_x64\protoc" target="tools\linux_x64\protoc" />
+    <file src="protoc\macosx_x86\protoc" target="tools\macosx_x86\protoc" />
+    <file src="protoc\macosx_x64\protoc" target="tools\macosx_x64\protoc" />
+    <file src="..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\api.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\descriptor.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\duration.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\empty.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\field_mask.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\source_context.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\struct.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\timestamp.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\type.proto" target="tools\google\protobuf" />
+    <file src="..\src\google\protobuf\wrappers.proto" target="tools\google\protobuf" />
+  </files>
+</package>
\ No newline at end of file

csharp/src/Google.Protobuf.Test/JsonFormatterTest.cs

@@ -321,7 +321,6 @@ namespace Google.Protobuf
         [TestCase("1970-01-01T00:00:00.001Z", 1000000)]
         [TestCase("1970-01-01T00:00:00.010Z", 10000000)]
         [TestCase("1970-01-01T00:00:00.100Z", 100000000)]
-        [TestCase("1970-01-01T00:00:00.100Z", 100000000)]
         [TestCase("1970-01-01T00:00:00.120Z", 120000000)]
         [TestCase("1970-01-01T00:00:00.123Z", 123000000)]
         [TestCase("1970-01-01T00:00:00.123400Z", 123400000)]

csharp/src/Google.Protobuf.Test/JsonTokenizerTest.cs

@@ -249,7 +249,6 @@ namespace Google.Protobuf
         [TestCase("[,", 1)]
         [TestCase("{", 1)]
         [TestCase("{,", 1)]
-        [TestCase("{", 1)]
         [TestCase("{[", 1)]
         [TestCase("{{", 1)]
         [TestCase("{0", 1)]

csharp/src/Google.Protobuf/Google.Protobuf.nuspec

@@ -23,7 +23,6 @@
       <group targetFramework="xamarin.ios" />
       <group targetFramework="monotouch" />
       <group targetFramework="monoandroid" />
-
       <!-- Dependencies for newer, more granular platforms (.NET Core etc) -->
       <group targetFramework="dotnet">
         <dependency id="System.Collections" version="4.0.0" />
@@ -49,18 +48,5 @@
     <file src="bin/ReleaseSigned/Google.Protobuf.pdb" target="lib/dotnet" />
     <file src="bin/ReleaseSigned/Google.Protobuf.xml" target="lib/dotnet" />
     <file src="**\*.cs" target="src" />
-    <file src="..\..\..\cmake\Release\protoc.exe" target="tools" />
-    <file src="..\..\..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\api.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\descriptor.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\duration.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\empty.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\field_mask.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\source_context.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\struct.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\timestamp.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\any.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\type.proto" target="tools\google\protobuf" />
-    <file src="..\..\..\src\google\protobuf\wrappers.proto" target="tools\google\protobuf" />
   </files>
 </package>
\ No newline at end of file

csharp/src/Google.Protobuf/Reflection/FieldDescriptor.cs

@@ -30,9 +30,8 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #endregion
 
-using System;
-using System.Linq;
 using Google.Protobuf.Compatibility;
+using System;
 
 namespace Google.Protobuf.Reflection
 {
@@ -41,20 +40,35 @@ namespace Google.Protobuf.Reflection
     /// </summary>
     public sealed class FieldDescriptor : DescriptorBase, IComparable<FieldDescriptor>
     {
-        private readonly FieldDescriptorProto proto;
         private EnumDescriptor enumType;
         private MessageDescriptor messageType;
-        private readonly MessageDescriptor containingType;
-        private readonly OneofDescriptor containingOneof;
         private FieldType fieldType;
         private readonly string propertyName; // Annoyingly, needed in Crosslink.
         private IFieldAccessor accessor;
 
+        /// <summary>
+        /// Get the field's containing message type.
+        /// </summary>
+        public MessageDescriptor ContainingType { get; }
+
+        /// <summary>
+        /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
+        /// </summary>
+        public OneofDescriptor ContainingOneof { get; }
+
+        /// <summary>
+        /// The effective JSON name for this field. This is usually the lower-camel-cased form of the field name,
+        /// but can be overridden using the <c>json_name</c> option in the .proto file.
+        /// </summary>
+        public string JsonName { get; }
+
+        internal FieldDescriptorProto Proto { get; }
+
         internal FieldDescriptor(FieldDescriptorProto proto, FileDescriptor file,
                                  MessageDescriptor parent, int index, string propertyName)
             : base(file, file.ComputeFullName(parent, proto.Name), index)
         {
-            this.proto = proto;
+            Proto = proto;
             if (proto.Type != 0)
             {
                 fieldType = GetFieldTypeFromProtoType(proto.Type);
@@ -64,7 +78,7 @@ namespace Google.Protobuf.Reflection
             {
                 throw new DescriptorValidationException(this, "Field numbers must be positive integers.");
             }
-            containingType = parent;
+            ContainingType = parent;
             // OneofIndex "defaults" to -1 due to a hack in FieldDescriptor.OnConstruction.
             if (proto.OneofIndex != -1)
             {
@@ -73,7 +87,7 @@ namespace Google.Protobuf.Reflection
                     throw new DescriptorValidationException(this,
                         $"FieldDescriptorProto.oneof_index is out of range for type {parent.Name}");
                 }
-                containingOneof = parent.Oneofs[proto.OneofIndex];
+                ContainingOneof = parent.Oneofs[proto.OneofIndex];
             }
 
             file.DescriptorPool.AddSymbol(this);
@@ -83,20 +97,14 @@ namespace Google.Protobuf.Reflection
             // We could trust the generated code and check whether the type of the property is
             // a MapField, but that feels a tad nasty.
             this.propertyName = propertyName;
+            JsonName =  Proto.JsonName == "" ? JsonFormatter.ToCamelCase(Proto.Name) : Proto.JsonName;
         }
+    
 
         /// <summary>
         /// The brief name of the descriptor's target.
         /// </summary>
-        public override string Name { get { return proto.Name; } }
-
-
-        /// <summary>
-        /// The json_name option of the descriptor's target.
-        /// </summary>
-        public string JsonName { get { return proto.JsonName == "" ? JsonFormatter.ToCamelCase(proto.Name) : proto.JsonName; } }
-
-        internal FieldDescriptorProto Proto { get { return proto; } }
+        public override string Name => Proto.Name;
 
         /// <summary>
         /// Returns the accessor for this field.
@@ -116,7 +124,7 @@ namespace Google.Protobuf.Reflection
         /// and this property will return null.
         /// </para>
         /// </remarks>
-        public IFieldAccessor Accessor { get { return accessor; } }
+        public IFieldAccessor Accessor => accessor;
         
         /// <summary>
         /// Maps a field type as included in the .proto file to a FieldType.
@@ -169,62 +177,32 @@ namespace Google.Protobuf.Reflection
         /// <summary>
         /// Returns <c>true</c> if this field is a repeated field; <c>false</c> otherwise.
         /// </summary>
-        public bool IsRepeated
-        {
-            get { return Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED; }
-        }
+        public bool IsRepeated => Proto.Label == FieldDescriptorProto.Types.Label.LABEL_REPEATED;
 
         /// <summary>
         /// Returns <c>true</c> if this field is a map field; <c>false</c> otherwise.
         /// </summary>
-        public bool IsMap
-        {
-            get { return fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry; }
-        }
+        public bool IsMap => fieldType == FieldType.Message && messageType.Proto.Options != null && messageType.Proto.Options.MapEntry;
 
         /// <summary>
         /// Returns <c>true</c> if this field is a packed, repeated field; <c>false</c> otherwise.
         /// </summary>
-        public bool IsPacked
-        {
+        public bool IsPacked => 
             // Note the || rather than && here - we're effectively defaulting to packed, because that *is*
             // the default in proto3, which is all we support. We may give the wrong result for the protos
             // within descriptor.proto, but that's okay, as they're never exposed and we don't use IsPacked
             // within the runtime.
-            get { return Proto.Options == null || Proto.Options.Packed; }
-        }        
-
-        /// <summary>
-        /// Get the field's containing message type.
-        /// </summary>
-        public MessageDescriptor ContainingType
-        {
-            get { return containingType; }
-        }
-
-        /// <summary>
-        /// Returns the oneof containing this field, or <c>null</c> if it is not part of a oneof.
-        /// </summary>
-        public OneofDescriptor ContainingOneof
-        {
-            get { return containingOneof; }
-        }
-
+            Proto.Options == null || Proto.Options.Packed;
+        
         /// <summary>
         /// Returns the type of the field.
         /// </summary>
-        public FieldType FieldType
-        {
-            get { return fieldType; }
-        }
+        public FieldType FieldType => fieldType;
 
         /// <summary>
         /// Returns the field number declared in the proto file.
         /// </summary>
-        public int FieldNumber
-        {
-            get { return Proto.Number; }
-        }
+        public int FieldNumber => Proto.Number;
 
         /// <summary>
         /// Compares this descriptor with another one, ordering in "canonical" order
@@ -234,7 +212,7 @@ namespace Google.Protobuf.Reflection
         /// </summary>
         public int CompareTo(FieldDescriptor other)
         {
-            if (other.containingType != containingType)
+            if (other.ContainingType != ContainingType)
             {
                 throw new ArgumentException("FieldDescriptors can only be compared to other FieldDescriptors " +
                                             "for fields of the same message type.");
@@ -337,14 +315,14 @@ namespace Google.Protobuf.Reflection
 
             File.DescriptorPool.AddFieldByNumber(this);
 
-            if (containingType != null && containingType.Proto.Options != null && containingType.Proto.Options.MessageSetWireFormat)
+            if (ContainingType != null && ContainingType.Proto.Options != null && ContainingType.Proto.Options.MessageSetWireFormat)
             {
                 throw new DescriptorValidationException(this, "MessageSet format is not supported.");
             }
-            accessor = CreateAccessor(propertyName);
+            accessor = CreateAccessor();
         }
 
-        private IFieldAccessor CreateAccessor(string propertyName)
+        private IFieldAccessor CreateAccessor()
         {
             // If we're given no property name, that's because we really don't want an accessor.
             // (At the moment, that means it's a map entry message...)
@@ -352,10 +330,10 @@ namespace Google.Protobuf.Reflection
             {
                 return null;
             }
-            var property = containingType.ClrType.GetProperty(propertyName);
+            var property = ContainingType.ClrType.GetProperty(propertyName);
             if (property == null)
             {
-                throw new DescriptorValidationException(this, $"Property {propertyName} not found in {containingType.ClrType}");
+                throw new DescriptorValidationException(this, $"Property {propertyName} not found in {ContainingType.ClrType}");
             }
             return IsMap ? new MapFieldAccessor(property, this)
                 : IsRepeated ? new RepeatedFieldAccessor(property, this)

objectivec/GPBCodedInputStream.h

@@ -109,6 +109,15 @@ NS_ASSUME_NONNULL_BEGIN
 /// or until an endgroup tag, whichever comes first.
 - (void)skipMessage;
 
+/// Check to see if the logical end of the stream has been reached.
+///
+/// This can return NO when there is no more data, but the current parsing
+/// expected more data.
+- (BOOL)isAtEnd;
+
+/// The offset into the stream.
+- (size_t)position;
+
 /// Verifies that the last call to @c -readTag returned the given tag value.
 /// This is used to verify that a nested group ended with the correct end tag.
 /// Throws @c NSParseErrorException if value does not match the last tag.

objectivec/GPBCodedInputStream.m

@@ -359,6 +359,14 @@ void GPBCodedInputStreamCheckLastTagWas(GPBCodedInputStreamState *state,
   }
 }
 
+- (BOOL)isAtEnd {
+  return GPBCodedInputStreamIsAtEnd(&state_);
+}
+
+- (size_t)position {
+  return state_.bufferPos;
+}
+
 - (double)readDouble {
   return GPBCodedInputStreamReadDouble(&state_);
 }

python/README.md

@@ -123,13 +123,5 @@ C++ Implementation
 The C++ implementation for Python messages is built as a Python extension to
 improve the overall protobuf Python performance.
 
-To use the C++ implementation, you need to:
-1) Install the C++ protobuf runtime library, please see instructions in the
-   parent directory.
-2) Export an environment variable:
-
-    $ export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp
-
-You must set this variable at runtime, before running your program, otherwise
-the pure-Python implementation will be used. In a future release, we will
-change the default so that C++ implementation is used whenever it is available.
+To use the C++ implementation, you need to install the C++ protobuf runtime
+library, please see instructions in the parent directory.

python/google/protobuf/internal/message_test.py

@@ -58,7 +58,11 @@ except ImportError:
   import unittest
 
 from google.protobuf.internal import _parameterized
+from google.protobuf import descriptor_pb2
+from google.protobuf import descriptor_pool
 from google.protobuf import map_unittest_pb2
+from google.protobuf import message_factory
+from google.protobuf import text_format
 from google.protobuf import unittest_pb2
 from google.protobuf import unittest_proto3_arena_pb2
 from google.protobuf.internal import api_implementation
@@ -1763,5 +1767,60 @@ class PackedFieldTest(unittest.TestCase):
                    b'\x70\x01')
     self.assertEqual(golden_data, message.SerializeToString())
 
+
+@unittest.skipIf(api_implementation.Type() != 'cpp',
+                 'explicit tests of the C++ implementation')
+class OversizeProtosTest(unittest.TestCase):
+
+  def setUp(self):
+    self.file_desc = """
+      name: "f/f.msg2"
+      package: "f"
+      message_type {
+        name: "msg1"
+        field {
+          name: "payload"
+          number: 1
+          label: LABEL_OPTIONAL
+          type: TYPE_STRING
+        }
+      }
+      message_type {
+        name: "msg2"
+        field {
+          name: "field"
+          number: 1
+          label: LABEL_OPTIONAL
+          type: TYPE_MESSAGE
+          type_name: "msg1"
+        }
+      }
+    """
+    pool = descriptor_pool.DescriptorPool()
+    desc = descriptor_pb2.FileDescriptorProto()
+    text_format.Parse(self.file_desc, desc)
+    pool.Add(desc)
+    self.proto_cls = message_factory.MessageFactory(pool).GetPrototype(
+        pool.FindMessageTypeByName('f.msg2'))
+    self.p = self.proto_cls()
+    self.p.field.payload = 'c' * (1024 * 1024 * 64 + 1)
+    self.p_serialized = self.p.SerializeToString()
+
+  def testAssertOversizeProto(self):
+    from google.protobuf.pyext._message import SetAllowOversizeProtos
+    SetAllowOversizeProtos(False)
+    q = self.proto_cls()
+    try:
+      q.ParseFromString(self.p_serialized)
+    except message.DecodeError as e:
+      self.assertEqual(str(e), 'Error parsing message')
+
+  def testSucceedOversizeProto(self):
+    from google.protobuf.pyext._message import SetAllowOversizeProtos
+    SetAllowOversizeProtos(True)
+    q = self.proto_cls()
+    q.ParseFromString(self.p_serialized)
+    self.assertEqual(self.p.field.payload, q.field.payload)
+
 if __name__ == '__main__':
   unittest.main()

python/google/protobuf/pyext/message.cc

@@ -1919,6 +1919,30 @@ static PyObject* CopyFrom(CMessage* self, PyObject* arg) {
   Py_RETURN_NONE;
 }
 
+// Protobuf has a 64MB limit built in, this variable will override this. Please
+// do not enable this unless you fully understand the implications: protobufs
+// must all be kept in memory at the same time, so if they grow too big you may
+// get OOM errors. The protobuf APIs do not provide any tools for processing
+// protobufs in chunks.  If you have protos this big you should break them up if
+// it is at all convenient to do so.
+static bool allow_oversize_protos = false;
+
+// Provide a method in the module to set allow_oversize_protos to a boolean
+// value. This method returns the newly value of allow_oversize_protos.
+static PyObject* SetAllowOversizeProtos(PyObject* m, PyObject* arg) {
+  if (!arg || !PyBool_Check(arg)) {
+    PyErr_SetString(PyExc_TypeError,
+                    "Argument to SetAllowOversizeProtos must be boolean");
+    return NULL;
+  }
+  allow_oversize_protos = PyObject_IsTrue(arg);
+  if (allow_oversize_protos) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
 static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
   const void* data;
   Py_ssize_t data_length;
@@ -1929,15 +1953,9 @@ static PyObject* MergeFromString(CMessage* self, PyObject* arg) {
   AssureWritable(self);
   io::CodedInputStream input(
       reinterpret_cast<const uint8*>(data), data_length);
-#if PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
-  // Protobuf has a 64MB limit built in, this code will override this. Please do
-  // not enable this unless you fully understand the implications: protobufs
-  // must all be kept in memory at the same time, so if they grow too big you
-  // may get OOM errors. The protobuf APIs do not provide any tools for
-  // processing protobufs in chunks.  If you have protos this big you should
-  // break them up if it is at all convenient to do so.
-  input.SetTotalBytesLimit(INT_MAX, INT_MAX);
-#endif  // PROTOBUF_PYTHON_ALLOW_OVERSIZE_PROTOS
+  if (allow_oversize_protos) {
+    input.SetTotalBytesLimit(INT_MAX, INT_MAX);
+  }
   PyDescriptorPool* pool = GetDescriptorPoolForMessage(self);
   input.SetExtensionRegistry(pool->pool, pool->message_factory);
   bool success = self->message->MergePartialFromCodedStream(&input);
@@ -3058,6 +3076,11 @@ bool InitProto2MessageModule(PyObject *m) {
 }  // namespace python
 }  // namespace protobuf
 
+static PyMethodDef ModuleMethods[] = {
+    {"SetAllowOversizeProtos",
+     (PyCFunction)google::protobuf::python::cmessage::SetAllowOversizeProtos,
+     METH_O, "Enable/disable oversize proto parsing."},
+};
 
 #if PY_MAJOR_VERSION >= 3
 static struct PyModuleDef _module = {
@@ -3065,7 +3088,7 @@ static struct PyModuleDef _module = {
   "_message",
   google::protobuf::python::module_docstring,
   -1,
-  NULL,
+  ModuleMethods,  /* m_methods */
   NULL,
   NULL,
   NULL,
@@ -3084,7 +3107,8 @@ extern "C" {
 #if PY_MAJOR_VERSION >= 3
     m = PyModule_Create(&_module);
 #else
-    m = Py_InitModule3("_message", NULL, google::protobuf::python::module_docstring);
+    m = Py_InitModule3("_message", ModuleMethods,
+                       google::protobuf::python::module_docstring);
 #endif
     if (m == NULL) {
       return INITFUNC_ERRORVAL;

python/setup.py

@@ -157,15 +157,30 @@ class test_conformance(_build_py):
     status = subprocess.check_call(cmd, shell=True)
 
 
+def get_option_from_sys_argv(option_str):
+  if option_str in sys.argv:
+    sys.argv.remove(option_str)
+    return True
+  return False
+
+
 if __name__ == '__main__':
   ext_module_list = []
-  cpp_impl = '--cpp_implementation'
   warnings_as_errors = '--warnings_as_errors'
-  if cpp_impl in sys.argv:
-    sys.argv.remove(cpp_impl)
+  if get_option_from_sys_argv('--cpp_implementation'):
+    # Link libprotobuf.a and libprotobuf-lite.a statically with the
+    # extension. Note that those libraries have to be compiled with
+    # -fPIC for this to work.
+    compile_static_ext = get_option_from_sys_argv('--compile_static_extension')
     extra_compile_args = ['-Wno-write-strings',
                           '-Wno-invalid-offsetof',
                           '-Wno-sign-compare']
+    libraries = ['protobuf']
+    extra_objects = None
+    if compile_static_ext:
+      libraries = None
+      extra_objects = ['../src/.libs/libprotobuf.a',
+                       '../src/.libs/libprotobuf-lite.a']
     test_conformance.target = 'test_python_cpp'
 
     if "clang" in os.popen('$CC --version 2> /dev/null').read():
@@ -176,16 +191,22 @@ if __name__ == '__main__':
       sys.argv.remove(warnings_as_errors)
 
     # C++ implementation extension
-    ext_module_list.append(
+    ext_module_list.extend([
         Extension(
             "google.protobuf.pyext._message",
             glob.glob('google/protobuf/pyext/*.cc'),
             include_dirs=[".", "../src"],
-            libraries=['protobuf'],
+            libraries=libraries,
+            extra_objects=extra_objects,
             library_dirs=['../src/.libs'],
             extra_compile_args=extra_compile_args,
-        )
-    )
+        ),
+        Extension(
+            "google.protobuf.internal._api_implementation",
+            glob.glob('google/protobuf/internal/api_implementation.cc'),
+            extra_compile_args=['-DPYTHON_PROTO2_CPP_IMPL_V2'],
+        ),
+    ])
     os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'cpp'
 
   # Keep this list of dependencies in sync with tox.ini.

src/google/protobuf/stubs/int128.cc

@@ -31,7 +31,7 @@
 #include <google/protobuf/stubs/int128.h>
 
 #include <iomanip>
-#include <iostream>  // NOLINT(readability/streams)
+#include <ostream>  // NOLINT(readability/streams)
 #include <sstream>
 
 namespace google {