Down integrate to GitHub (#6893)

js/map.js

@@ -208,14 +208,14 @@ jspb.Map.ArrayIteratorIterable_.prototype.next = function() {
   }
 };
 
-
 if (typeof(Symbol) != 'undefined') {
-/** @override */
-jspb.Map.ArrayIteratorIterable_.prototype[Symbol.iterator] = function() {
-  return this;
-};
+  /** @override */
+  jspb.Map.ArrayIteratorIterable_.prototype[Symbol.iterator] = function() {
+    return this;
+  };
 }
 
+
 /**
  * Returns the map's length (number of key/value pairs).
  * @return {number}

python/google/protobuf/descriptor.py

@@ -873,9 +873,14 @@ class FileDescriptor(DescriptorBase):
                 syntax=None, pool=None):
       # FileDescriptor() is called from various places, not only from generated
       # files, to register dynamic proto files and messages.
-      if serialized_pb:
-        # TODO(amauryfa): use the pool passed as argument. This will work only
-        # for C++-implemented DescriptorPools.
+      # pylint: disable=g-explicit-bool-comparison
+      if serialized_pb == '':
+        # Cpp generated code must be linked in if serialized_pb is ''
+        try:
+          return _message.default_pool.FindFileByName(name)
+        except KeyError:
+          raise RuntimeError('Please link in cpp generated lib for %s' % (name))
+      elif serialized_pb:
         return _message.default_pool.AddSerializedFile(serialized_pb)
       else:
         return super(FileDescriptor, cls).__new__(cls)

python/google/protobuf/pyext/message.cc

@@ -216,15 +216,41 @@ static PyObject* New(PyTypeObject* type,
   }
 
   // Check dict['DESCRIPTOR']
-  PyObject* py_descriptor = PyDict_GetItem(dict, kDESCRIPTOR);
-  if (py_descriptor == NULL) {
+  PyObject* descriptor_or_name = PyDict_GetItem(dict, kDESCRIPTOR);
+  if (descriptor_or_name == nullptr) {
     PyErr_SetString(PyExc_TypeError, "Message class has no DESCRIPTOR");
     return NULL;
   }
-  if (!PyObject_TypeCheck(py_descriptor, &PyMessageDescriptor_Type)) {
-    PyErr_Format(PyExc_TypeError, "Expected a message Descriptor, got %s",
-                 py_descriptor->ob_type->tp_name);
-    return NULL;
+
+  Py_ssize_t name_size;
+  char* full_name;
+  const Descriptor* message_descriptor;
+  PyObject* py_descriptor;
+
+  if (PyObject_TypeCheck(descriptor_or_name, &PyMessageDescriptor_Type)) {
+    py_descriptor = descriptor_or_name;
+    message_descriptor = PyMessageDescriptor_AsDescriptor(py_descriptor);
+    if (message_descriptor == nullptr) {
+      return nullptr;
+    }
+  } else {
+    if (PyString_AsStringAndSize(descriptor_or_name, &full_name, &name_size) <
+        0) {
+      return nullptr;
+    }
+    message_descriptor =
+        GetDefaultDescriptorPool()->pool->FindMessageTypeByName(
+            std::string(full_name, name_size));
+    if (message_descriptor == nullptr) {
+      PyErr_Format(PyExc_KeyError,
+                   "Can not find message descriptor %s "
+                   "from pool",
+                   full_name);
+      return nullptr;
+    }
+    py_descriptor = PyMessageDescriptor_FromDescriptor(message_descriptor);
+    // reset the dict['DESCRIPTOR'] to py_descriptor.
+    PyDict_SetItem(dict, kDESCRIPTOR, py_descriptor);
   }
 
   // Messages have no __dict__
@@ -236,11 +262,6 @@ static PyObject* New(PyTypeObject* type,
   // Build the arguments to the base metaclass.
   // We change the __bases__ classes.
   ScopedPyObjectPtr new_args;
-  const Descriptor* message_descriptor =
-      PyMessageDescriptor_AsDescriptor(py_descriptor);
-  if (message_descriptor == NULL) {
-    return NULL;
-  }
 
   if (WKT_classes == NULL) {
     ScopedPyObjectPtr well_known_types(PyImport_ImportModule(

python/google/protobuf/service_reflection.py

@@ -38,6 +38,12 @@ compiler at compile-time.
 
 __author__ = 'petar@google.com (Petar Petrov)'
 
+from google.protobuf.internal import api_implementation
+
+if api_implementation.Type() == 'cpp':
+  # pylint: disable=g-import-not-at-top
+  from google.protobuf.pyext import _message
+
 
 class GeneratedServiceType(type):
 
@@ -76,9 +82,15 @@ class GeneratedServiceType(type):
     # when a service class is subclassed.
     if GeneratedServiceType._DESCRIPTOR_KEY not in dictionary:
       return
+
     descriptor = dictionary[GeneratedServiceType._DESCRIPTOR_KEY]
+    if isinstance(descriptor, str):
+      descriptor = _message.default_pool.FindServiceByName(descriptor)
+      dictionary[GeneratedServiceType._DESCRIPTOR_KEY] = descriptor
+
     service_builder = _ServiceBuilder(descriptor)
     service_builder.BuildService(cls)
+    cls.DESCRIPTOR = descriptor
 
 
 class GeneratedServiceStubType(GeneratedServiceType):
@@ -101,12 +113,16 @@ class GeneratedServiceStubType(GeneratedServiceType):
         dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object
         describing this protocol service type.
     """
+    descriptor = dictionary.get(cls._DESCRIPTOR_KEY)
+    if isinstance(descriptor, str):
+      descriptor = _message.default_pool.FindServiceByName(descriptor)
+      dictionary[GeneratedServiceStubType._DESCRIPTOR_KEY] = descriptor
     super(GeneratedServiceStubType, cls).__init__(name, bases, dictionary)
     # Don't do anything if this class doesn't have a descriptor. This happens
     # when a service stub is subclassed.
     if GeneratedServiceStubType._DESCRIPTOR_KEY not in dictionary:
       return
-    descriptor = dictionary[GeneratedServiceStubType._DESCRIPTOR_KEY]
+
     service_stub_builder = _ServiceStubBuilder(descriptor)
     service_stub_builder.BuildServiceStub(cls)
 

src/google/protobuf/compiler/command_line_interface.cc

@@ -811,6 +811,7 @@ void CommandLineInterface::AllowPlugins(const std::string& exe_name_prefix) {
   plugin_prefix_ = exe_name_prefix;
 }
 
+
 int CommandLineInterface::Run(int argc, const char* const argv[]) {
   Clear();
   switch (ParseArguments(argc, argv)) {

src/google/protobuf/compiler/csharp/csharp_message.cc

@@ -152,7 +152,9 @@ void MessageGenerator::Generate(io::Printer* printer) {
 
     // a read-only property for fast
     // retrieval of the set in IsInitialized
-    printer->Print(vars, "private pb::ExtensionSet<$class_name$> _Extensions { get { return _extensions; } }\n");
+    printer->Print(vars,
+                   "private pb::ExtensionSet<$class_name$> _Extensions { get { "
+                   "return _extensions; } }\n");
   }
 
   for (int i = 0; i < has_bit_field_count_; i++) {

src/google/protobuf/compiler/java/java_helpers.h

@@ -160,6 +160,7 @@ inline bool MultipleJavaFiles(const FileDescriptor* descriptor,
   return descriptor->options().java_multiple_files();
 }
 
+
 // Returns true if `descriptor` will be written to its own .java file.
 // `immutable` should be set to true if we're generating for the immutable API.
 template <typename Descriptor>

src/google/protobuf/compiler/java/java_name_resolver.cc

@@ -210,9 +210,9 @@ std::string ClassNameResolver::GetClassName(const FileDescriptor* descriptor,
 // or outer class name.
 std::string ClassNameResolver::GetClassFullName(
     const std::string& name_without_package, const FileDescriptor* file,
-    bool immutable, bool multiple_files) {
+    bool immutable, bool is_own_file) {
   std::string result;
-  if (multiple_files) {
+  if (is_own_file) {
     result = FileJavaPackage(file, immutable);
   } else {
     result = GetClassName(file, immutable);
@@ -242,7 +242,7 @@ std::string ClassNameResolver::GetClassName(const ServiceDescriptor* descriptor,
                                             bool immutable) {
   return GetClassFullName(ClassNameWithoutPackage(descriptor, immutable),
                           descriptor->file(), immutable,
-                          MultipleJavaFiles(descriptor->file(), immutable));
+                          IsOwnFile(descriptor, immutable));
 }
 
 // Get the Java Class style full name of a message.

src/google/protobuf/compiler/java/java_name_resolver.h

@@ -108,7 +108,7 @@ class ClassNameResolver {
   // or outer class name.
   std::string GetClassFullName(const std::string& name_without_package,
                                const FileDescriptor* file, bool immutable,
-                               bool multiple_files);
+                               bool is_own_file);
   // Get the Java Class style full name of a message.
   std::string GetJavaClassFullName(const std::string& name_without_package,
                                    const FileDescriptor* file, bool immutable);

src/google/protobuf/compiler/python/python_generator.h

@@ -166,6 +166,7 @@ class PROTOC_EXPORT Generator : public CodeGenerator {
   mutable const FileDescriptor* file_;  // Set in Generate().  Under mutex_.
   mutable std::string file_descriptor_serialized_;
   mutable io::Printer* printer_;  // Set in Generate().  Under mutex_.
+  mutable bool pure_python_workable_;
 
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Generator);
 };

src/google/protobuf/descriptor.cc

@@ -4602,7 +4602,6 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto,
 
   AddSymbol(result->full_name(), parent, result->name(), proto, Symbol(result));
 
-
   for (int i = 0; i < proto.reserved_range_size(); i++) {
     const DescriptorProto_ReservedRange& range1 = proto.reserved_range(i);
     for (int j = i + 1; j < proto.reserved_range_size(); j++) {

src/google/protobuf/parse_context.cc

@@ -197,22 +197,25 @@ const char* EpsCopyInputStream::SkipFallback(const char* ptr, int size) {
 }
 
 const char* EpsCopyInputStream::ReadStringFallback(const char* ptr, int size,
-                                                   std::string* s) {
-  s->clear();
-  // TODO(gerbens) assess security. At the moment its parity with
-  // CodedInputStream but it allows a payload to reserve large memory.
+                                                   std::string* str) {
+  str->clear();
   if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) {
-    s->reserve(size);
+    // Reserve the string up to a static safe size. If strings are bigger than
+    // this we proceed by growing the string as needed. This protects against
+    // malicious payloads making protobuf hold on to a lot of memory.
+    str->reserve(str->size() + std::min<int>(size, kSafeStringSize));
   }
-  return AppendStringFallback(ptr, size, s);
+  return AppendSize(ptr, size,
+                    [str](const char* p, int s) { str->append(p, s); });
 }
 
 const char* EpsCopyInputStream::AppendStringFallback(const char* ptr, int size,
                                                      std::string* str) {
-  // TODO(gerbens) assess security. At the moment its parity with
-  // CodedInputStream but it allows a payload to reserve large memory.
   if (PROTOBUF_PREDICT_TRUE(size <= buffer_end_ - ptr + limit_)) {
-    str->reserve(size);
+    // Reserve the string up to a static safe size. If strings are bigger than
+    // this we proceed by growing the string as needed. This protects against
+    // malicious payloads making protobuf hold on to a lot of memory.
+    str->reserve(str->size() + std::min<int>(size, kSafeStringSize));
   }
   return AppendSize(ptr, size,
                     [str](const char* p, int s) { str->append(p, s); });

src/google/protobuf/parse_context.h

@@ -271,6 +271,9 @@ class PROTOBUF_EXPORT EpsCopyInputStream {
   // DoneFallback.
   uint32 last_tag_minus_1_ = 0;
   int overall_limit_ = INT_MAX;  // Overall limit independent of pushed limits.
+  // Pretty random large number that seems like a safe allocation on most
+  // systems. TODO(gerbens) do we need to set this as build flag?
+  enum { kSafeStringSize = 50000000 };
 
   std::pair<const char*, bool> DoneFallback(const char* ptr, int d);
   const char* Next(int overrun, int d);