Merge pull request #3988 from acozzette/down-integrate

Integrated internal changes from Google

Makefile.am

@@ -204,6 +204,7 @@ java_EXTRA_DIST=
   java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java             \
   java/core/src/main/java/com/google/protobuf/AbstractParser.java                  \
   java/core/src/main/java/com/google/protobuf/AbstractProtobufList.java            \
+  java/core/src/main/java/com/google/protobuf/Android.java                         \
   java/core/src/main/java/com/google/protobuf/BlockingRpcChannel.java              \
   java/core/src/main/java/com/google/protobuf/BlockingService.java                 \
   java/core/src/main/java/com/google/protobuf/BooleanArrayList.java                \

cmake/libprotobuf-lite.cmake

@@ -4,6 +4,7 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc
+  ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.cc
   ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.cc
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.cc
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.cc
@@ -31,6 +32,7 @@ set(libprotobuf_lite_includes
   ${protobuf_source_dir}/src/google/protobuf/arenastring.h
   ${protobuf_source_dir}/src/google/protobuf/extension_set.h
   ${protobuf_source_dir}/src/google/protobuf/generated_message_util.h
+  ${protobuf_source_dir}/src/google/protobuf/implicit_weak_message.h
   ${protobuf_source_dir}/src/google/protobuf/io/coded_stream.h
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream.h
   ${protobuf_source_dir}/src/google/protobuf/io/zero_copy_stream_impl_lite.h

conformance/failure_list_cpp.txt

@@ -54,4 +54,3 @@ Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.SINT64
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT32
 Required.Proto2.ProtobufInput.PrematureEofInPackedField.UINT64
-

conformance/failure_list_java.txt

@@ -44,4 +44,4 @@ Required.Proto3.JsonInput.StringFieldNotAString
 Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
 Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
 Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
-Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
\ No newline at end of file
+Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE

csharp/src/Google.Protobuf/WellKnownTypes/Any.cs

@@ -154,17 +154,18 @@ namespace Google.Protobuf.WellKnownTypes {
     public const int TypeUrlFieldNumber = 1;
     private string typeUrl_ = "";
     /// <summary>
-    /// A URL/resource name whose content describes the type of the
-    /// serialized protocol buffer message.
+    /// A URL/resource name that uniquely identifies the type of the serialized
+    /// protocol buffer message. The last segment of the URL's path must represent
+    /// the fully qualified name of the type (as in
+    /// `path/google.protobuf.Duration`). The name should be in a canonical form
+    /// (e.g., leading "." is not accepted).
     ///
-    /// For URLs which use the scheme `http`, `https`, or no scheme, the
-    /// following restrictions and interpretations apply:
+    /// In practice, teams usually precompile into the binary all types that they
+    /// expect it to use in the context of Any. However, for URLs which use the
+    /// scheme `http`, `https`, or no scheme, one can optionally set up a type
+    /// server that maps type URLs to message definitions as follows:
     ///
     /// * If no scheme is provided, `https` is assumed.
-    /// * The last segment of the URL's path must represent the fully
-    ///   qualified name of the type (as in `path/google.protobuf.Duration`).
-    ///   The name should be in a canonical form (e.g., leading "." is
-    ///   not accepted).
     /// * An HTTP GET on the URL must yield a [google.protobuf.Type][]
     ///   value in binary format, or produce an error.
     /// * Applications are allowed to cache lookup results based on the
@@ -173,6 +174,10 @@ namespace Google.Protobuf.WellKnownTypes {
     ///   on changes to types. (Use versioned type names to manage
     ///   breaking changes.)
     ///
+    /// Note: this functionality is not currently available in the official
+    /// protobuf release, and it is not used for type URLs beginning with
+    /// type.googleapis.com.
+    ///
     /// Schemes other than `http`, `https` (or the empty scheme) might be
     /// used with implementation specific semantics.
     /// </summary>

csharp/src/Google.Protobuf/WellKnownTypes/Timestamp.cs

@@ -112,8 +112,8 @@ namespace Google.Protobuf.WellKnownTypes {
   /// to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime)
   /// with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one
   /// can use the Joda Time's [`ISODateTimeFormat.dateTime()`](
-  /// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--)
-  /// to obtain a formatter capable of generating timestamps in this format.
+  /// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--
+  /// ) to obtain a formatter capable of generating timestamps in this format.
   /// </summary>
   public sealed partial class Timestamp : pb::IMessage<Timestamp> {
     private static readonly pb::MessageParser<Timestamp> _parser = new pb::MessageParser<Timestamp>(() => new Timestamp());

java/core/src/main/java/com/google/protobuf/Android.java

+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package com.google.protobuf;
+
+final class Android {
+
+  private static final Class<?> MEMORY_CLASS = getClassForName("libcore.io.Memory");
+  private static final boolean IS_ROBOLECTRIC =
+      getClassForName("org.robolectric.Robolectric") != null;
+
+  /** Returns {@code true} if running on an Android device. */
+  static boolean isOnAndroidDevice() {
+    return MEMORY_CLASS != null && !IS_ROBOLECTRIC;
+  }
+
+  /** Returns the memory class or {@code null} if not on Android device. */
+  static Class<?> getMemoryClass() {
+    return MEMORY_CLASS;
+  }
+
+  @SuppressWarnings("unchecked")
+  private static <T> Class<T> getClassForName(String name) {
+    try {
+      return (Class<T>) Class.forName(name);
+    } catch (Throwable e) {
+      return null;
+    }
+  }
+}

java/core/src/main/java/com/google/protobuf/ByteString.java

@@ -124,14 +124,8 @@ public abstract class ByteString implements Iterable<Byte>, Serializable {
 
   private static final ByteArrayCopier byteArrayCopier;
   static {
-    boolean isAndroid = true;
-    try {
-      Class.forName("android.content.Context");
-    } catch (ClassNotFoundException e) {
-      isAndroid = false;
-    }
-
-    byteArrayCopier = isAndroid ? new SystemByteArrayCopier() : new ArraysByteArrayCopier();
+    byteArrayCopier =
+        Android.isOnAndroidDevice() ? new SystemByteArrayCopier() : new ArraysByteArrayCopier();
   }
 
   /**

java/core/src/main/java/com/google/protobuf/FieldSet.java

@@ -102,6 +102,11 @@ final class FieldSet<FieldDescriptorType extends
   @SuppressWarnings("rawtypes")
   private static final FieldSet DEFAULT_INSTANCE = new FieldSet(true);
 
+  /** Returns {@code true} if empty, {@code false} otherwise. */
+  boolean isEmpty() {
+    return fields.isEmpty();
+  }
+
   /** Make this FieldSet immutable from this point forward. */
   @SuppressWarnings("unchecked")
   public void makeImmutable() {

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -504,11 +504,8 @@ public abstract class GeneratedMessageLite<
             extends GeneratedMessageLite<MessageType, BuilderType>
             implements ExtendableMessageOrBuilder<MessageType, BuilderType> {
 
-    /**
-     * Represents the set of extensions on this message. For use by generated
-     * code only.
-     */
-    protected FieldSet<ExtensionDescriptor> extensions = FieldSet.newFieldSet();
+    /** Represents the set of extensions on this message. For use by generated code only. */
+    protected FieldSet<ExtensionDescriptor> extensions = FieldSet.emptySet();
 
     @SuppressWarnings("unchecked")
     protected final void mergeExtensionFields(final MessageType other) {
@@ -578,7 +575,11 @@ public abstract class GeneratedMessageLite<
       if (unknown) {  // Unknown field or wrong wire type.  Skip.
         return parseUnknownField(tag, input);
       }
-      
+
+      if (extensions.isImmutable()) {
+        extensions = extensions.clone();
+      }
+
       if (packed) {
         int length = input.readRawVarint32();
         int limit = input.pushLimit(length);
@@ -942,12 +943,6 @@ public abstract class GeneratedMessageLite<
       implements ExtendableMessageOrBuilder<MessageType, BuilderType> {
     protected ExtendableBuilder(MessageType defaultInstance) {
       super(defaultInstance);
-
-      // TODO(dweis): This is kind of an unnecessary clone since we construct a
-      //     new instance in the parent constructor which makes the extensions
-      //     immutable. This extra allocation shouldn't matter in practice
-      //     though.
-      instance.extensions = instance.extensions.clone();
     }
 
     // For immutable message conversion.
@@ -966,6 +961,15 @@ public abstract class GeneratedMessageLite<
       instance.extensions = instance.extensions.clone();
     }
 
+    private FieldSet<ExtensionDescriptor> ensureExtensionsAreMutable() {
+      FieldSet<ExtensionDescriptor> extensions = instance.extensions;
+      if (extensions.isImmutable()) {
+        extensions = extensions.clone();
+        instance.extensions = extensions;
+      }
+      return extensions;
+    }
+
     @Override
     public final MessageType buildPartial() {
       if (isBuilt) {
@@ -1024,7 +1028,8 @@ public abstract class GeneratedMessageLite<
 
       verifyExtensionContainingType(extensionLite);
       copyOnWrite();
-      instance.extensions.setField(extensionLite.descriptor, extensionLite.toFieldSetType(value));
+      ensureExtensionsAreMutable()
+          .setField(extensionLite.descriptor, extensionLite.toFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -1037,8 +1042,9 @@ public abstract class GeneratedMessageLite<
 
       verifyExtensionContainingType(extensionLite);
       copyOnWrite();
-      instance.extensions.setRepeatedField(
-          extensionLite.descriptor, index, extensionLite.singularToFieldSetType(value));
+      ensureExtensionsAreMutable()
+          .setRepeatedField(
+              extensionLite.descriptor, index, extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -1051,8 +1057,8 @@ public abstract class GeneratedMessageLite<
 
       verifyExtensionContainingType(extensionLite);
       copyOnWrite();
-      instance.extensions.addRepeatedField(
-          extensionLite.descriptor, extensionLite.singularToFieldSetType(value));
+      ensureExtensionsAreMutable()
+          .addRepeatedField(extensionLite.descriptor, extensionLite.singularToFieldSetType(value));
       return (BuilderType) this;
     }
 
@@ -1063,7 +1069,7 @@ public abstract class GeneratedMessageLite<
 
       verifyExtensionContainingType(extensionLite);
       copyOnWrite();
-      instance.extensions.clearField(extensionLite.descriptor);
+      ensureExtensionsAreMutable().clearField(extensionLite.descriptor);
       return (BuilderType) this;
     }
   }

java/core/src/main/java/com/google/protobuf/TextFormat.java

@@ -1223,6 +1223,22 @@ public final class TextFormat {
     PARSER.merge(input, builder);
   }
 
+  /**
+   * Parse a text-format message from {@code input}.
+   *
+   * @return the parsed message, guaranteed initialized
+   */
+  public static <T extends Message> T parse(final CharSequence input,
+                                            final Class<T> protoClass)
+                                            throws ParseException {
+    Message.Builder builder =
+        Internal.getDefaultInstance(protoClass).newBuilderForType();
+    merge(input, builder);
+    @SuppressWarnings("unchecked")
+    T output = (T) builder.build();
+    return output;
+  }
+
   /**
    * Parse a text-format message from {@code input} and merge the contents
    * into {@code builder}.  Extensions will be recognized if they are
@@ -1248,6 +1264,25 @@ public final class TextFormat {
     PARSER.merge(input, extensionRegistry, builder);
   }
 
+  /**
+   * Parse a text-format message from {@code input}.  Extensions will be
+   * recognized if they are registered in {@code extensionRegistry}.
+   *
+   * @return the parsed message, guaranteed initialized
+   */
+  public static <T extends Message> T parse(
+      final CharSequence input,
+      final ExtensionRegistry extensionRegistry,
+      final Class<T> protoClass)
+      throws ParseException {
+    Message.Builder builder =
+        Internal.getDefaultInstance(protoClass).newBuilderForType();
+    merge(input, extensionRegistry, builder);
+    @SuppressWarnings("unchecked")
+    T output = (T) builder.build();
+    return output;
+  }
+
 
   /**
    * Parser for text-format proto2 instances. This class is thread-safe.

java/core/src/main/java/com/google/protobuf/UnsafeUtil.java

@@ -385,15 +385,6 @@ final class UnsafeUtil {
   }
 
 
-  @SuppressWarnings("unchecked")
-  private static <T> Class<T> getClassForName(String name) {
-    try {
-      return (Class<T>) Class.forName(name);
-    } catch (Throwable e) {
-      return null;
-    }
-  }
-
   /** Finds the address field within a direct {@link Buffer}. */
   private static Field bufferAddressField() {
     return field(Buffer.class, "address", long.class);

java/core/src/test/java/com/google/protobuf/AbstractMessageTest.java

@@ -30,6 +30,9 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.TestUtil.TEST_REQUIRED_INITIALIZED;
+import static com.google.protobuf.TestUtil.TEST_REQUIRED_UNINITIALIZED;
+
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import protobuf_unittest.UnittestOptimizeFor.TestOptimizedForSize;
 import protobuf_unittest.UnittestProto;
@@ -346,11 +349,6 @@ public class AbstractMessageTest extends TestCase {
   // -----------------------------------------------------------------
   // Tests for isInitialized().
 
-  private static final TestRequired TEST_REQUIRED_UNINITIALIZED =
-    TestRequired.getDefaultInstance();
-  private static final TestRequired TEST_REQUIRED_INITIALIZED =
-    TestRequired.newBuilder().setA(1).setB(2).setC(3).build();
-
   public void testIsInitialized() throws Exception {
     TestRequired.Builder builder = TestRequired.newBuilder();
     AbstractMessageWrapper.Builder abstractBuilder =
@@ -380,7 +378,7 @@ public class AbstractMessageTest extends TestCase {
     builder.setOptionalMessage(TEST_REQUIRED_UNINITIALIZED);
     assertFalse(abstractBuilder.isInitialized());
     assertEquals(
-        "optional_message.a, optional_message.b, optional_message.c",
+        "optional_message.b, optional_message.c",
         abstractBuilder.getInitializationErrorString());
 
     builder.setOptionalMessage(TEST_REQUIRED_INITIALIZED);
@@ -390,7 +388,7 @@ public class AbstractMessageTest extends TestCase {
     builder.addRepeatedMessage(TEST_REQUIRED_UNINITIALIZED);
     assertFalse(abstractBuilder.isInitialized());
     assertEquals(
-        "repeated_message[0].a, repeated_message[0].b, repeated_message[0].c",
+        "repeated_message[0].b, repeated_message[0].c",
         abstractBuilder.getInitializationErrorString());
 
     builder.setRepeatedMessage(0, TEST_REQUIRED_INITIALIZED);

java/core/src/test/java/com/google/protobuf/TestUtil.java

@@ -231,6 +231,7 @@ import protobuf_unittest.UnittestProto.TestAllTypesOrBuilder;
 import protobuf_unittest.UnittestProto.TestOneof2;
 import protobuf_unittest.UnittestProto.TestPackedExtensions;
 import protobuf_unittest.UnittestProto.TestPackedTypes;
+import protobuf_unittest.UnittestProto.TestRequired;
 import protobuf_unittest.UnittestProto.TestUnpackedTypes;
 import java.io.File;
 import java.io.IOException;
@@ -252,6 +253,11 @@ import junit.framework.Assert;
 public final class TestUtil {
   private TestUtil() {}
 
+  public static final TestRequired TEST_REQUIRED_UNINITIALIZED =
+      TestRequired.newBuilder().setA(1).buildPartial();
+  public static final TestRequired TEST_REQUIRED_INITIALIZED =
+      TestRequired.newBuilder().setA(1).setB(2).setC(3).build();
+
   /** Helper to convert a String to ByteString. */
   static ByteString toBytes(String str) {
     return ByteString.copyFrom(str.getBytes(Internal.UTF_8));

java/core/src/test/java/com/google/protobuf/TextFormatTest.java

@@ -30,6 +30,9 @@
 
 package com.google.protobuf;
 
+import static com.google.protobuf.TestUtil.TEST_REQUIRED_INITIALIZED;
+import static com.google.protobuf.TestUtil.TEST_REQUIRED_UNINITIALIZED;
+
 import com.google.protobuf.Descriptors.Descriptor;
 import com.google.protobuf.Descriptors.FieldDescriptor;
 import com.google.protobuf.TextFormat.Parser.SingularOverwritePolicy;
@@ -42,6 +45,7 @@ import protobuf_unittest.UnittestProto.TestAllTypes;
 import protobuf_unittest.UnittestProto.TestAllTypes.NestedMessage;
 import protobuf_unittest.UnittestProto.TestEmptyMessage;
 import protobuf_unittest.UnittestProto.TestOneof2;
+import protobuf_unittest.UnittestProto.TestRequired;
 import proto2_wireformat_unittest.UnittestMsetWireFormat.TestMessageSet;
 import java.io.StringReader;
 import java.util.List;
@@ -326,19 +330,58 @@ public class TextFormatTest extends TestCase {
 
   // =================================================================
 
-  public void testParse() throws Exception {
+  public void testMerge() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge(allFieldsSetText, builder);
     TestUtil.assertAllFieldsSet(builder.build());
   }
 
-  public void testParseReader() throws Exception {
+  public void testParse() throws Exception {
+    TestUtil.assertAllFieldsSet(
+        TextFormat.parse(allFieldsSetText, TestAllTypes.class));
+  }
+
+  public void testMergeInitialized() throws Exception {
+    TestRequired.Builder builder = TestRequired.newBuilder();
+    TextFormat.merge(TEST_REQUIRED_INITIALIZED.toString(), builder);
+    assertEquals(TEST_REQUIRED_INITIALIZED.toString(),
+                 builder.buildPartial().toString());
+    assertTrue(builder.isInitialized());
+  }
+
+  public void testParseInitialized() throws Exception {
+    TestRequired parsed =
+        TextFormat.parse(TEST_REQUIRED_INITIALIZED.toString(),
+                         TestRequired.class);
+    assertEquals(TEST_REQUIRED_INITIALIZED.toString(), parsed.toString());
+    assertTrue(parsed.isInitialized());
+  }
+
+  public void testMergeUninitialized() throws Exception {
+    TestRequired.Builder builder = TestRequired.newBuilder();
+    TextFormat.merge(TEST_REQUIRED_UNINITIALIZED.toString(), builder);
+    assertEquals(TEST_REQUIRED_UNINITIALIZED.toString(),
+                 builder.buildPartial().toString());
+    assertFalse(builder.isInitialized());
+  }
+
+  public void testParseUninitialized() throws Exception {
+    try {
+      TextFormat.parse(TEST_REQUIRED_UNINITIALIZED.toString(),
+                       TestRequired.class);
+      fail("Expected UninitializedMessageException.");
+    } catch (UninitializedMessageException e) {
+      assertEquals("Message missing required fields: b, c", e.getMessage());
+    }
+  }
+
+  public void testMergeReader() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge(new StringReader(allFieldsSetText), builder);
     TestUtil.assertAllFieldsSet(builder.build());
   }
 
-  public void testParseExtensions() throws Exception {
+  public void testMergeExtensions() throws Exception {
     TestAllExtensions.Builder builder = TestAllExtensions.newBuilder();
     TextFormat.merge(allExtensionsSetText,
                      TestUtil.getExtensionRegistry(),
@@ -346,7 +389,14 @@ public class TextFormatTest extends TestCase {
     TestUtil.assertAllExtensionsSet(builder.build());
   }
 
-  public void testParseCompatibility() throws Exception {
+  public void testParseExtensions() throws Exception {
+    TestUtil.assertAllExtensionsSet(
+        TextFormat.parse(allExtensionsSetText,
+                         TestUtil.getExtensionRegistry(),
+                         TestAllExtensions.class));
+  }
+
+  public void testMergeAndParseCompatibility() throws Exception {
     String original = "repeated_float: inf\n" +
                       "repeated_float: -inf\n" +
                       "repeated_float: nan\n" +
@@ -371,21 +421,29 @@ public class TextFormatTest extends TestCase {
                         "repeated_double: Infinity\n" +
                         "repeated_double: -Infinity\n" +
                         "repeated_double: NaN\n";
+
+    // Test merge().
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge(original, builder);
     assertEquals(canonical, builder.build().toString());
+
+    // Test parse().
+    assertEquals(canonical,
+                 TextFormat.parse(original, TestAllTypes.class).toString());
   }
 
-  public void testParseExotic() throws Exception {
+  public void testMergeAndParseExotic() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge(exoticText, builder);
 
     // Too lazy to check things individually.  Don't try to debug this
     // if testPrintExotic() is failing.
     assertEquals(canonicalExoticText, builder.build().toString());
+    assertEquals(canonicalExoticText,
+                 TextFormat.parse(exoticText, TestAllTypes.class).toString());
   }
 
-  public void testParseMessageSet() throws Exception {
+  public void testMergeMessageSet() throws Exception {
     ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance();
     extensionRegistry.add(TestMessageSetExtension1.messageSetExtension);
     extensionRegistry.add(TestMessageSetExtension2.messageSetExtension);
@@ -411,7 +469,7 @@ public class TextFormatTest extends TestCase {
       TestMessageSetExtension1.messageSetExtension).getI());
   }
 
-  public void testParseMessageSetWithOverwriteForbidden() throws Exception {
+  public void testMergeMessageSetWithOverwriteForbidden() throws Exception {
     ExtensionRegistry extensionRegistry = ExtensionRegistry.newInstance();
     extensionRegistry.add(TestMessageSetExtension1.messageSetExtension);
     extensionRegistry.add(TestMessageSetExtension2.messageSetExtension);
@@ -438,20 +496,20 @@ public class TextFormatTest extends TestCase {
     }
   }
 
-  public void testParseNumericEnum() throws Exception {
+  public void testMergeNumericEnum() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge("optional_nested_enum: 2", builder);
     assertEquals(TestAllTypes.NestedEnum.BAR, builder.getOptionalNestedEnum());
   }
 
-  public void testParseAngleBrackets() throws Exception {
+  public void testMergeAngleBrackets() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge("OptionalGroup: < a: 1 >", builder);
     assertTrue(builder.hasOptionalGroup());
     assertEquals(1, builder.getOptionalGroup().getA());
   }
 
-  public void testParseComment() throws Exception {
+  public void testMergeComment() throws Exception {
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     TextFormat.merge(
       "# this is a comment\n" +
@@ -463,6 +521,7 @@ public class TextFormatTest extends TestCase {
   }
 
   private void assertParseError(String error, String text) {
+    // Test merge().
     TestAllTypes.Builder builder = TestAllTypes.newBuilder();
     try {
       TextFormat.merge(text, TestUtil.getExtensionRegistry(), builder);
@@ -470,6 +529,15 @@ public class TextFormatTest extends TestCase {
     } catch (TextFormat.ParseException e) {
       assertEquals(error, e.getMessage());
     }
+
+    // Test parse().
+    try {
+      TextFormat.parse(
+          text, TestUtil.getExtensionRegistry(), TestAllTypes.class);
+      fail("Expected parse exception.");
+    } catch (TextFormat.ParseException e) {
+      assertEquals(error, e.getMessage());
+    }
   }
 
 

java/util/src/main/java/com/google/protobuf/util/JsonFormat.java

@@ -105,7 +105,7 @@ public class JsonFormat {
   public static Printer printer() {
     return new Printer(
         TypeRegistry.getEmptyTypeRegistry(), false, Collections.<FieldDescriptor>emptySet(),
-        false, false);
+        false, false, false);
   }
 
   /**
@@ -125,18 +125,21 @@ public class JsonFormat {
     private Set<FieldDescriptor> includingDefaultValueFields;
     private final boolean preservingProtoFieldNames;
     private final boolean omittingInsignificantWhitespace;
+    private final boolean printingEnumsAsInts;
 
     private Printer(
         TypeRegistry registry,
         boolean alwaysOutputDefaultValueFields,
         Set<FieldDescriptor> includingDefaultValueFields,
         boolean preservingProtoFieldNames,
-        boolean omittingInsignificantWhitespace) {
+        boolean omittingInsignificantWhitespace,
+        boolean printingEnumsAsInts) {
       this.registry = registry;
       this.alwaysOutputDefaultValueFields = alwaysOutputDefaultValueFields;
       this.includingDefaultValueFields = includingDefaultValueFields;
       this.preservingProtoFieldNames = preservingProtoFieldNames;
       this.omittingInsignificantWhitespace = omittingInsignificantWhitespace;
+      this.printingEnumsAsInts = printingEnumsAsInts;
     }
 
     /**
@@ -154,7 +157,8 @@ public class JsonFormat {
           alwaysOutputDefaultValueFields,
           includingDefaultValueFields,
           preservingProtoFieldNames,
-          omittingInsignificantWhitespace);
+          omittingInsignificantWhitespace,
+          printingEnumsAsInts);
     }
 
     /**
@@ -170,7 +174,31 @@ public class JsonFormat {
           true,
           Collections.<FieldDescriptor>emptySet(),
           preservingProtoFieldNames,
-          omittingInsignificantWhitespace);
+          omittingInsignificantWhitespace,
+          printingEnumsAsInts);
+    }
+
+    /**
+     * Creates a new {@link Printer} that will print enum field values as integers instead of as
+     * string.
+     * The new Printer clones all other configurations from the current
+     * {@link Printer}.
+     */
+    public Printer printingEnumsAsInts() {
+      checkUnsetPrintingEnumsAsInts();
+      return new Printer(
+          registry,
+          alwaysOutputDefaultValueFields,
+          Collections.<FieldDescriptor>emptySet(),
+          preservingProtoFieldNames,
+          omittingInsignificantWhitespace,
+          true);
+    }
+
+    private void checkUnsetPrintingEnumsAsInts() {
+      if (printingEnumsAsInts) {
+        throw new IllegalStateException("JsonFormat printingEnumsAsInts has already been set.");
+      }
     }
 
     /**
@@ -191,7 +219,8 @@ public class JsonFormat {
           false,
           fieldsToAlwaysOutput,
           preservingProtoFieldNames,
-          omittingInsignificantWhitespace);
+          omittingInsignificantWhitespace,
+          printingEnumsAsInts);
     }
 
     private void checkUnsetIncludingDefaultValueFields() {
@@ -213,7 +242,8 @@ public class JsonFormat {
           alwaysOutputDefaultValueFields,
           includingDefaultValueFields,
           true,
-          omittingInsignificantWhitespace);
+          omittingInsignificantWhitespace,
+          printingEnumsAsInts);
     }
 
 
@@ -240,7 +270,8 @@ public class JsonFormat {
           alwaysOutputDefaultValueFields,
           includingDefaultValueFields,
           preservingProtoFieldNames,
-          true);
+          true,
+          printingEnumsAsInts);
     }
 
     /**
@@ -259,7 +290,8 @@ public class JsonFormat {
               includingDefaultValueFields,
               preservingProtoFieldNames,
               output,
-              omittingInsignificantWhitespace)
+              omittingInsignificantWhitespace,
+              printingEnumsAsInts)
           .print(message);
     }
 
@@ -416,7 +448,7 @@ public class JsonFormat {
        */
       public Builder add(Iterable<Descriptor> messageTypes) {
         if (types == null) {
-          throw new IllegalStateException("A TypeRegistry.Builer can only be used once.");
+          throw new IllegalStateException("A TypeRegistry.Builder can only be used once.");
         }
         for (Descriptor type : messageTypes) {
           addFile(type.getFile());
@@ -570,6 +602,7 @@ public class JsonFormat {
     private final boolean alwaysOutputDefaultValueFields;
     private final Set<FieldDescriptor> includingDefaultValueFields;
     private final boolean preservingProtoFieldNames;
+    private final boolean printingEnumsAsInts;
     private final TextGenerator generator;
     // We use Gson to help handle string escapes.
     private final Gson gson;
@@ -586,11 +619,13 @@ public class JsonFormat {
         Set<FieldDescriptor> includingDefaultValueFields,
         boolean preservingProtoFieldNames,
         Appendable jsonOutput,
-        boolean omittingInsignificantWhitespace) {
+        boolean omittingInsignificantWhitespace,
+        boolean printingEnumsAsInts) {
       this.registry = registry;
       this.alwaysOutputDefaultValueFields = alwaysOutputDefaultValueFields;
       this.includingDefaultValueFields = includingDefaultValueFields;
       this.preservingProtoFieldNames = preservingProtoFieldNames;
+      this.printingEnumsAsInts = printingEnumsAsInts;
       this.gson = GsonHolder.DEFAULT_GSON;
       // json format related properties, determined by printerType
       if (omittingInsignificantWhitespace) {
@@ -1069,7 +1104,7 @@ public class JsonFormat {
               generator.print("\"");
             }
           } else {
-            if (((EnumValueDescriptor) value).getIndex() == -1) {
+            if (printingEnumsAsInts || ((EnumValueDescriptor) value).getIndex() == -1) {
               generator.print(String.valueOf(((EnumValueDescriptor) value).getNumber()));
             } else {
               generator.print("\"" + ((EnumValueDescriptor) value).getName() + "\"");

java/util/src/main/java/com/google/protobuf/util/Timestamps.java

@@ -74,6 +74,11 @@ public final class Timestamps {
   public static final Timestamp MAX_VALUE =
       Timestamp.newBuilder().setSeconds(TIMESTAMP_SECONDS_MAX).setNanos(999999999).build();
 
+  /**
+   * A constant holding the {@link Timestamp} of epoch time, {@code 1970-01-01T00:00:00.000000000Z}.
+   */
+  public static final Timestamp EPOCH = Timestamp.newBuilder().setSeconds(0).setNanos(0).build();
+
   private static final ThreadLocal<SimpleDateFormat> timestampFormat =
       new ThreadLocal<SimpleDateFormat>() {
         @Override
@@ -358,10 +363,12 @@ public final class Timestamps {
   static Timestamp normalizedTimestamp(long seconds, int nanos) {
     if (nanos <= -NANOS_PER_SECOND || nanos >= NANOS_PER_SECOND) {
       seconds = checkedAdd(seconds, nanos / NANOS_PER_SECOND);
-      nanos %= NANOS_PER_SECOND;
+      nanos = (int) (nanos % NANOS_PER_SECOND);
     }
     if (nanos < 0) {
-      nanos += NANOS_PER_SECOND; // no overflow since nanos is negative (and we're adding)
+      nanos =
+          (int)
+              (nanos + NANOS_PER_SECOND); // no overflow since nanos is negative (and we're adding)
       seconds = checkedSubtract(seconds, 1);
     }
     Timestamp timestamp = Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build();

java/util/src/test/java/com/google/protobuf/util/JsonFormatTest.java

@@ -1174,6 +1174,14 @@ public class JsonFormatTest extends TestCase {
     JsonFormat.parser().ignoringUnknownFields().merge(json, builder);
   }
 
+  public void testParserIntegerEnumValue() throws Exception {
+    TestAllTypes.Builder actualBuilder = TestAllTypes.newBuilder();
+    mergeFromJson("{\n" + "  \"optionalNestedEnum\": 2\n" + "}", actualBuilder);
+
+    TestAllTypes expected = TestAllTypes.newBuilder().setOptionalNestedEnum(NestedEnum.BAZ).build();
+    assertEquals(expected, actualBuilder.build());
+  }
+
   public void testCustomJsonName() throws Exception {
     TestCustomJsonName message = TestCustomJsonName.newBuilder().setValue(12345).build();
     assertEquals("{\n" + "  \"@value\": 12345\n" + "}", JsonFormat.printer().print(message));
@@ -1441,6 +1449,13 @@ public class JsonFormatTest extends TestCase {
     assertEquals(54321, builder.getOptionalInt32());
   }
 
+  public void testPrintingEnumsAsInts() throws Exception {
+    TestAllTypes message = TestAllTypes.newBuilder().setOptionalNestedEnum(NestedEnum.BAR).build();
+    assertEquals(
+        "{\n" + "  \"optionalNestedEnum\": 1\n" + "}",
+        JsonFormat.printer().printingEnumsAsInts().print(message));
+  }
+
   public void testOmittingInsignificantWhiteSpace() throws Exception {
     TestAllTypes message = TestAllTypes.newBuilder().setOptionalInt32(12345).build();
     assertEquals(

js/binary/arith.js

@@ -221,7 +221,7 @@ jspb.arith.UInt64.prototype.mul = function(a) {
  * Divide a 64-bit number by a 32-bit number to produce a
  * 64-bit quotient and a 32-bit remainder.
  * @param {number} _divisor
- * @return {Array.<jspb.arith.UInt64>} array of [quotient, remainder],
+ * @return {Array<jspb.arith.UInt64>} array of [quotient, remainder],
  * unless divisor is 0, in which case an empty array is returned.
  */
 jspb.arith.UInt64.prototype.div = function(_divisor) {

js/binary/decoder.js

@@ -58,7 +58,7 @@ goog.require('jspb.utils');
  * @param {?jspb.BinaryDecoder=} opt_decoder
  * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=}
  *     opt_next The decoder method to use for next().
- * @param {?Array.<number|boolean|string>=} opt_elements
+ * @param {?Array<number|boolean|string>=} opt_elements
  * @constructor
  * @struct
  */
@@ -92,7 +92,7 @@ jspb.BinaryIterator = function(opt_decoder, opt_next, opt_elements) {
  * @param {?jspb.BinaryDecoder=} opt_decoder
  * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=}
  *     opt_next The decoder method to use for next().
- * @param {?Array.<number|boolean|string>=} opt_elements
+ * @param {?Array<number|boolean|string>=} opt_elements
  * @private
  */
 jspb.BinaryIterator.prototype.init_ =
@@ -112,7 +112,7 @@ jspb.BinaryIterator.prototype.init_ =
 
 /**
  * Global pool of BinaryIterator instances.
- * @private {!Array.<!jspb.BinaryIterator>}
+ * @private {!Array<!jspb.BinaryIterator>}
  */
 jspb.BinaryIterator.instanceCache_ = [];
 
@@ -123,7 +123,7 @@ jspb.BinaryIterator.instanceCache_ = [];
  * @param {?jspb.BinaryDecoder=} opt_decoder
  * @param {?function(this:jspb.BinaryDecoder):(number|boolean|string)=}
  *     opt_next The decoder method to use for next().
- * @param {?Array.<number|boolean|string>=} opt_elements
+ * @param {?Array<number|boolean|string>=} opt_elements
  * @return {!jspb.BinaryIterator}
  */
 jspb.BinaryIterator.alloc = function(opt_decoder, opt_next, opt_elements) {
@@ -274,7 +274,7 @@ jspb.BinaryDecoder = function(opt_bytes, opt_start, opt_length) {
 
 /**
  * Global pool of BinaryDecoder instances.
- * @private {!Array.<!jspb.BinaryDecoder>}
+ * @private {!Array<!jspb.BinaryDecoder>}
  */
 jspb.BinaryDecoder.instanceCache_ = [];
 

js/binary/encoder.js

@@ -51,7 +51,7 @@ goog.require('jspb.utils');
  * @struct
  */
 jspb.BinaryEncoder = function() {
-  /** @private {!Array.<number>} */
+  /** @private {!Array<number>} */
   this.buffer_ = [];
 };
 
@@ -65,7 +65,7 @@ jspb.BinaryEncoder.prototype.length = function() {
 
 
 /**
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryEncoder.prototype.end = function() {
   var buffer = this.buffer_;

js/binary/reader.js

@@ -97,7 +97,7 @@ jspb.BinaryReader = function(opt_bytes, opt_start, opt_length) {
 
   /**
    * User-defined reader callbacks.
-   * @private {Object.<string, function(!jspb.BinaryReader):*>}
+   * @private {Object<string, function(!jspb.BinaryReader):*>}
    */
   this.readCallbacks_ = null;
 };
@@ -105,7 +105,7 @@ jspb.BinaryReader = function(opt_bytes, opt_start, opt_length) {
 
 /**
  * Global pool of BinaryReader instances.
- * @private {!Array.<!jspb.BinaryReader>}
+ * @private {!Array<!jspb.BinaryReader>}
  */
 jspb.BinaryReader.instanceCache_ = [];
 
@@ -992,7 +992,7 @@ jspb.BinaryReader.prototype.readPackedField_ = function(decodeMethod) {
 /**
  * Reads a packed int32 field, which consists of a length header and a list of
  * signed varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedInt32 = function() {
   return this.readPackedField_(this.decoder_.readSignedVarint32);
@@ -1002,7 +1002,7 @@ jspb.BinaryReader.prototype.readPackedInt32 = function() {
 /**
  * Reads a packed int32 field, which consists of a length header and a list of
  * signed varints. Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedInt32String = function() {
   return this.readPackedField_(this.decoder_.readSignedVarint32String);
@@ -1012,7 +1012,7 @@ jspb.BinaryReader.prototype.readPackedInt32String = function() {
 /**
  * Reads a packed int64 field, which consists of a length header and a list of
  * signed varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedInt64 = function() {
   return this.readPackedField_(this.decoder_.readSignedVarint64);
@@ -1022,7 +1022,7 @@ jspb.BinaryReader.prototype.readPackedInt64 = function() {
 /**
  * Reads a packed int64 field, which consists of a length header and a list of
  * signed varints. Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedInt64String = function() {
   return this.readPackedField_(this.decoder_.readSignedVarint64String);
@@ -1032,7 +1032,7 @@ jspb.BinaryReader.prototype.readPackedInt64String = function() {
 /**
  * Reads a packed uint32 field, which consists of a length header and a list of
  * unsigned varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedUint32 = function() {
   return this.readPackedField_(this.decoder_.readUnsignedVarint32);
@@ -1042,7 +1042,7 @@ jspb.BinaryReader.prototype.readPackedUint32 = function() {
 /**
  * Reads a packed uint32 field, which consists of a length header and a list of
  * unsigned varints. Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedUint32String = function() {
   return this.readPackedField_(this.decoder_.readUnsignedVarint32String);
@@ -1052,7 +1052,7 @@ jspb.BinaryReader.prototype.readPackedUint32String = function() {
 /**
  * Reads a packed uint64 field, which consists of a length header and a list of
  * unsigned varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedUint64 = function() {
   return this.readPackedField_(this.decoder_.readUnsignedVarint64);
@@ -1062,7 +1062,7 @@ jspb.BinaryReader.prototype.readPackedUint64 = function() {
 /**
  * Reads a packed uint64 field, which consists of a length header and a list of
  * unsigned varints. Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedUint64String = function() {
   return this.readPackedField_(this.decoder_.readUnsignedVarint64String);
@@ -1072,7 +1072,7 @@ jspb.BinaryReader.prototype.readPackedUint64String = function() {
 /**
  * Reads a packed sint32 field, which consists of a length header and a list of
  * zigzag varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedSint32 = function() {
   return this.readPackedField_(this.decoder_.readZigzagVarint32);
@@ -1082,7 +1082,7 @@ jspb.BinaryReader.prototype.readPackedSint32 = function() {
 /**
  * Reads a packed sint64 field, which consists of a length header and a list of
  * zigzag varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedSint64 = function() {
   return this.readPackedField_(this.decoder_.readZigzagVarint64);
@@ -1092,7 +1092,7 @@ jspb.BinaryReader.prototype.readPackedSint64 = function() {
 /**
  * Reads a packed sint64 field, which consists of a length header and a list of
  * zigzag varints.  Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedSint64String = function() {
   return this.readPackedField_(this.decoder_.readZigzagVarint64String);
@@ -1102,7 +1102,7 @@ jspb.BinaryReader.prototype.readPackedSint64String = function() {
 /**
  * Reads a packed fixed32 field, which consists of a length header and a list
  * of unsigned 32-bit ints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedFixed32 = function() {
   return this.readPackedField_(this.decoder_.readUint32);
@@ -1112,7 +1112,7 @@ jspb.BinaryReader.prototype.readPackedFixed32 = function() {
 /**
  * Reads a packed fixed64 field, which consists of a length header and a list
  * of unsigned 64-bit ints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedFixed64 = function() {
   return this.readPackedField_(this.decoder_.readUint64);
@@ -1122,7 +1122,7 @@ jspb.BinaryReader.prototype.readPackedFixed64 = function() {
 /**
  * Reads a packed fixed64 field, which consists of a length header and a list
  * of unsigned 64-bit ints.  Returns a list of strings.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedFixed64String = function() {
   return this.readPackedField_(this.decoder_.readUint64String);
@@ -1132,7 +1132,7 @@ jspb.BinaryReader.prototype.readPackedFixed64String = function() {
 /**
  * Reads a packed sfixed32 field, which consists of a length header and a list
  * of 32-bit ints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedSfixed32 = function() {
   return this.readPackedField_(this.decoder_.readInt32);
@@ -1142,7 +1142,7 @@ jspb.BinaryReader.prototype.readPackedSfixed32 = function() {
 /**
  * Reads a packed sfixed64 field, which consists of a length header and a list
  * of 64-bit ints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedSfixed64 = function() {
   return this.readPackedField_(this.decoder_.readInt64);
@@ -1152,7 +1152,7 @@ jspb.BinaryReader.prototype.readPackedSfixed64 = function() {
 /**
  * Reads a packed sfixed64 field, which consists of a length header and a list
  * of 64-bit ints.  Returns a list of strings.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedSfixed64String = function() {
   return this.readPackedField_(this.decoder_.readInt64String);
@@ -1162,7 +1162,7 @@ jspb.BinaryReader.prototype.readPackedSfixed64String = function() {
 /**
  * Reads a packed float field, which consists of a length header and a list of
  * floats.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedFloat = function() {
   return this.readPackedField_(this.decoder_.readFloat);
@@ -1172,7 +1172,7 @@ jspb.BinaryReader.prototype.readPackedFloat = function() {
 /**
  * Reads a packed double field, which consists of a length header and a list of
  * doubles.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedDouble = function() {
   return this.readPackedField_(this.decoder_.readDouble);
@@ -1182,7 +1182,7 @@ jspb.BinaryReader.prototype.readPackedDouble = function() {
 /**
  * Reads a packed bool field, which consists of a length header and a list of
  * unsigned varints.
- * @return {!Array.<boolean>}
+ * @return {!Array<boolean>}
  */
 jspb.BinaryReader.prototype.readPackedBool = function() {
   return this.readPackedField_(this.decoder_.readBool);
@@ -1192,7 +1192,7 @@ jspb.BinaryReader.prototype.readPackedBool = function() {
 /**
  * Reads a packed enum field, which consists of a length header and a list of
  * unsigned varints.
- * @return {!Array.<number>}
+ * @return {!Array<number>}
  */
 jspb.BinaryReader.prototype.readPackedEnum = function() {
   return this.readPackedField_(this.decoder_.readEnum);
@@ -1202,7 +1202,7 @@ jspb.BinaryReader.prototype.readPackedEnum = function() {
 /**
  * Reads a packed varint hash64 field, which consists of a length header and a
  * list of varint hash64s.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedVarintHash64 = function() {
   return this.readPackedField_(this.decoder_.readVarintHash64);
@@ -1212,7 +1212,7 @@ jspb.BinaryReader.prototype.readPackedVarintHash64 = function() {
 /**
  * Reads a packed fixed hash64 field, which consists of a length header and a
  * list of fixed hash64s.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.BinaryReader.prototype.readPackedFixedHash64 = function() {
   return this.readPackedField_(this.decoder_.readFixedHash64);

js/binary/utils.js

@@ -431,7 +431,7 @@ jspb.utils.joinHash64 = function(bitsLow, bitsHigh) {
 
 /**
  * Individual digits for number->string conversion.
- * @const {!Array.<string>}
+ * @const {!Array<string>}
  */
 jspb.utils.DIGITS = [
   '0', '1', '2', '3', '4', '5', '6', '7',
@@ -554,10 +554,10 @@ jspb.utils.hash64ToDecimalString = function(hash, signed) {
 /**
  * Converts an array of 8-character hash strings into their decimal
  * representations.
- * @param {!Array.<string>} hashes The array of hash strings to convert.
+ * @param {!Array<string>} hashes The array of hash strings to convert.
  * @param {boolean} signed True if we should treat the hash string as encoding
  *     a signed integer.
- * @return {!Array.<string>}
+ * @return {!Array<string>}
  */
 jspb.utils.hash64ArrayToDecimalStrings = function(hashes, signed) {
   var result = new Array(hashes.length);
@@ -972,7 +972,7 @@ jspb.utils.byteSourceToUint8Array = function(data) {
   }
 
   if (data.constructor === Array) {
-    data = /** @type {!Array.<number>} */(data);
+    data = /** @type {!Array<number>} */(data);
     return /** @type {!Uint8Array} */(new Uint8Array(data));
   }
 

js/binary/utils_test.js

@@ -657,7 +657,7 @@ describe('binaryUtilsTest', function() {
     // Converting Uint8Arrays into Uint8Arrays should be a no-op.
     assertEquals(sourceBytes, convert(sourceBytes));
 
-    // Converting Array.<numbers> into Uint8Arrays should work.
+    // Converting Array<numbers> into Uint8Arrays should work.
     check(convert(sourceData));
 
     // Converting ArrayBuffers into Uint8Arrays should work.

js/commonjs/export_testdeps.js

@@ -12,6 +12,7 @@
 goog.provide('jspb.ExportTestDeps');
 
 goog.require('goog.crypt.base64');
+goog.require('goog.testing.PropertyReplacer');
 goog.require('jspb.arith.Int64');
 goog.require('jspb.arith.UInt64');
 goog.require('jspb.BinaryEncoder');

js/debug.js

@@ -74,15 +74,22 @@ jspb.debug.dump = function(message) {
  */
 jspb.debug.dump_ = function(thing) {
   var type = goog.typeOf(thing);
+  var message = thing;  // Copy because we don't want type inference on thing.
   if (type == 'number' || type == 'string' || type == 'boolean' ||
       type == 'null' || type == 'undefined') {
     return thing;
   }
+  if (typeof Uint8Array !== 'undefined') {
+    // Will fail on IE9, where Uint8Array doesn't exist.
+    if (message instanceof Uint8Array) {
+      return thing;
+    }
+  }
+
   if (type == 'array') {
     goog.asserts.assertArray(thing);
     return goog.array.map(thing, jspb.debug.dump_);
   }
-  var message = thing;  // Copy because we don't want type inference on thing.
   goog.asserts.assert(message instanceof jspb.Message,
       'Only messages expected: ' + thing);
   var ctor = message.constructor;

js/debug_test.js

@@ -41,6 +41,8 @@ goog.require('proto.jspb.test.IsExtension');
 goog.require('proto.jspb.test.Simple1');
 
 
+// CommonJS-LoadFromFile: testbinary_pb
+goog.require('proto.jspb.test.TestAllTypes');
 
 describe('debugTest', function() {
   it('testSimple1', function() {
@@ -74,6 +76,15 @@ describe('debugTest', function() {
     }, jspb.debug.dump(message));
   });
 
+  it('testBytes', function() {
+    if (COMPILED || typeof Uint8Array == 'undefined') {
+      return;
+    }
+    var message = new proto.jspb.test.TestAllTypes();
+    var bytes = new Uint8Array(4);
+    message.setOptionalBytes(bytes);
+    assertEquals(jspb.debug.dump(message)['optionalBytes'], bytes);
+  });
 
   it('testExtensions', function() {
     if (COMPILED) {

js/message.js

@@ -205,6 +205,16 @@ goog.define('jspb.Message.GENERATE_TO_STRING', true);
 goog.define('jspb.Message.ASSUME_LOCAL_ARRAYS', false);
 
 
+// TODO(jakubvrana): Turn this off by default.
+/**
+ * @define {boolean} Disabling the serialization of empty trailing fields
+ *     reduces the size of serialized protos. The price is an extra iteration of
+ *     the proto before serialization. This is enabled by default to be
+ *     backwards compatible. Projects are advised to turn this flag always off.
+ */
+goog.define('jspb.Message.SERIALIZE_EMPTY_TRAILING_FIELDS', true);
+
+
 /**
  * @define {boolean} Turning on this flag does NOT change the behavior of JSPB
  *     and only affects private internal state. It may, however, break some
@@ -212,8 +222,8 @@ goog.define('jspb.Message.ASSUME_LOCAL_ARRAYS', false);
  *     mutates its internal state.
  *     Projects are advised to turn this flag always on.
  */
-goog.define('jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS', COMPILED);
-// TODO(b/19419436) Turn this on by default.
+goog.define('jspb.Message.MINIMIZE_MEMORY_ALLOCATIONS', true);
+// TODO(b/19419436): Delete this flag.
 
 
 /**
@@ -277,6 +287,13 @@ jspb.Message.prototype.messageId_;
 jspb.Message.prototype.convertedFloatingPointFields_;
 
 
+/**
+ * Repeated fields numbers.
+ * @protected {?Array<number>|undefined}
+ */
+jspb.Message.prototype.repeatedFields;
+
+
 /**
  * The xid of this proto type (The same for all instances of a proto). Provides
  * a way to identify a proto by stable obfuscated name.
@@ -323,6 +340,18 @@ jspb.Message.getIndex_ = function(msg, fieldNumber) {
 };
 
 
+/**
+ * Returns the tag number based on the index in msg.array.
+ * @param {!jspb.Message} msg Message for which we're calculating an index.
+ * @param {number} index The tag number.
+ * @return {number} The field number.
+ * @private
+ */
+jspb.Message.getFieldNumber_ = function(msg, index) {
+  return index - msg.arrayIndexOffset_;
+};
+
+
 /**
  * Initializes a JsPb Message.
  * @param {!jspb.Message} msg The JsPb proto to modify.
@@ -353,6 +382,13 @@ jspb.Message.initialize = function(
   jspb.Message.initPivotAndExtensionObject_(msg, suggestedPivot);
   msg.convertedFloatingPointFields_ = {};
 
+  if (!jspb.Message.SERIALIZE_EMPTY_TRAILING_FIELDS) {
+    // TODO(jakubvrana): This is same for all instances, move to prototype.
+    // TODO(jakubvrana): There are indexOf calls on this in serializtion,
+    // consider switching to a set.
+    msg.repeatedFields = repeatedFields;
+  }
+
   if (repeatedFields) {
     for (var i = 0; i < repeatedFields.length; i++) {
       var fieldNumber = repeatedFields[i];
@@ -376,8 +412,9 @@ jspb.Message.initialize = function(
   if (opt_oneofFields && opt_oneofFields.length) {
     // Compute the oneof case for each union. This ensures only one value is
     // set in the union.
-    goog.array.forEach(
-        opt_oneofFields, goog.partial(jspb.Message.computeOneofCase, msg));
+    for (var i = 0; i < opt_oneofFields.length; i++) {
+      jspb.Message.computeOneofCase(msg, opt_oneofFields[i]);
+    }
   }
 };
 
@@ -428,7 +465,7 @@ jspb.Message.initPivotAndExtensionObject_ = function(msg, suggestedPivot) {
     // in Safari on iOS 8. See the description of CL/86511464 for details.
     if (obj && typeof obj == 'object' && !jspb.Message.isArray_(obj) &&
         !(jspb.Message.SUPPORTS_UINT8ARRAY_ && obj instanceof Uint8Array)) {
-      msg.pivot_ = foundIndex - msg.arrayIndexOffset_;
+      msg.pivot_ = jspb.Message.getFieldNumber_(msg, foundIndex);
       msg.extensionObject_ = obj;
       return;
     }
@@ -1043,14 +1080,15 @@ jspb.Message.computeOneofCase = function(msg, oneof) {
   var oneofField;
   var oneofValue;
 
-  goog.array.forEach(oneof, function(fieldNumber) {
+  for (var i = 0; i < oneof.length; i++) {
+    var fieldNumber = oneof[i];
     var value = jspb.Message.getField(msg, fieldNumber);
-    if (goog.isDefAndNotNull(value)) {
+    if (value != null) {
       oneofField = fieldNumber;
       oneofValue = value;
       jspb.Message.setField(msg, fieldNumber, undefined);
     }
-  });
+  }
 
   if (oneofField) {
     // NB: We know the value is unique, so we can call jspb.Message.setField
@@ -1241,7 +1279,7 @@ jspb.Message.addToRepeatedWrapperField = function(
  *     dead code removal.
  * @param {boolean=} opt_includeInstance Whether to include the JSPB instance
  *     for transitional soy proto support: http://goto/soy-param-migration
- * @return {!Object.<string, Object>} A map of proto or Soy objects.
+ * @return {!Object<string, Object>} A map of proto or Soy objects.
  * @template T
  */
 jspb.Message.toMap = function(
@@ -1318,7 +1356,7 @@ jspb.Message.prototype.toString = function() {
 
 /**
  * Gets the value of the extension field from the extended object.
- * @param {jspb.ExtensionFieldInfo.<T>} fieldInfo Specifies the field to get.
+ * @param {jspb.ExtensionFieldInfo<T>} fieldInfo Specifies the field to get.
  * @return {T} The value of the field.
  * @template T
  */
@@ -1705,7 +1743,11 @@ jspb.Message.registry_ = {};
  * non-MessageSet. We special case MessageSet so that we do not need
  * to goog.require MessageSet from classes that extends MessageSet.
  *
- * @type {!Object.<number, jspb.ExtensionFieldInfo>}
+ * @type {!Object<number, jspb.ExtensionFieldInfo>}
  */
 jspb.Message.messageSetExtensions = {};
+
+/**
+ * @type {!Object<number, jspb.ExtensionFieldBinaryInfo>}
+ */
 jspb.Message.messageSetExtensionsBinary = {};

js/message_test.js

@@ -34,6 +34,7 @@ goog.setTestOnly();
 
 goog.require('goog.json');
 goog.require('goog.string');
+goog.require('goog.testing.PropertyReplacer');
 goog.require('goog.testing.asserts');
 goog.require('goog.userAgent');
 
@@ -85,6 +86,16 @@ goog.require('proto.jspb.test.TestExtensionsMessage');
 
 
 describe('Message test suite', function() {
+  var stubs = new goog.testing.PropertyReplacer();
+
+  beforeEach(function() {
+    stubs.set(jspb.Message, 'SERIALIZE_EMPTY_TRAILING_FIELDS', false);
+  });
+
+  afterEach(function() {
+    stubs.reset();
+  });
+
   it('testEmptyProto', function() {
     var empty1 = new proto.jspb.test.Empty([]);
     var empty2 = new proto.jspb.test.Empty([]);

objectivec/google/protobuf/Any.pbobjc.h

@@ -135,17 +135,18 @@ typedef GPB_ENUM(GPBAny_FieldNumber) {
 @interface GPBAny : GPBMessage
 
 /**
- * A URL/resource name whose content describes the type of the
- * serialized protocol buffer message.
+ * A URL/resource name that uniquely identifies the type of the serialized
+ * protocol buffer message. The last segment of the URL's path must represent
+ * the fully qualified name of the type (as in
+ * `path/google.protobuf.Duration`). The name should be in a canonical form
+ * (e.g., leading "." is not accepted).
  *
- * For URLs which use the scheme `http`, `https`, or no scheme, the
- * following restrictions and interpretations apply:
+ * In practice, teams usually precompile into the binary all types that they
+ * expect it to use in the context of Any. However, for URLs which use the
+ * scheme `http`, `https`, or no scheme, one can optionally set up a type
+ * server that maps type URLs to message definitions as follows:
  *
  * * If no scheme is provided, `https` is assumed.
- * * The last segment of the URL's path must represent the fully
- *   qualified name of the type (as in `path/google.protobuf.Duration`).
- *   The name should be in a canonical form (e.g., leading "." is
- *   not accepted).
  * * An HTTP GET on the URL must yield a [google.protobuf.Type][]
  *   value in binary format, or produce an error.
  * * Applications are allowed to cache lookup results based on the
@@ -154,6 +155,10 @@ typedef GPB_ENUM(GPBAny_FieldNumber) {
  *   on changes to types. (Use versioned type names to manage
  *   breaking changes.)
  *
+ * Note: this functionality is not currently available in the official
+ * protobuf release, and it is not used for type URLs beginning with
+ * type.googleapis.com.
+ *
  * Schemes other than `http`, `https` (or the empty scheme) might be
  * used with implementation specific semantics.
  **/

objectivec/google/protobuf/Timestamp.pbobjc.h

@@ -126,8 +126,8 @@ typedef GPB_ENUM(GPBTimestamp_FieldNumber) {
  * to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime)
  * with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one
  * can use the Joda Time's [`ISODateTimeFormat.dateTime()`](
- * http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--)
- * to obtain a formatter capable of generating timestamps in this format.
+ * http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--
+ * ) to obtain a formatter capable of generating timestamps in this format.
  **/
 @interface GPBTimestamp : GPBMessage
 

python/google/protobuf/descriptor.py

@@ -41,8 +41,8 @@ from google.protobuf.internal import api_implementation
 _USE_C_DESCRIPTORS = False
 if api_implementation.Type() == 'cpp':
   # Used by MakeDescriptor in cpp mode
+  import binascii
   import os
-  import uuid
   from google.protobuf.pyext import _message
   _USE_C_DESCRIPTORS = getattr(_message, '_USE_C_DESCRIPTORS', False)
 
@@ -952,7 +952,7 @@ def MakeDescriptor(desc_proto, package='', build_file_if_cpp=True,
     # imported ones. We need to specify a file name so the descriptor pool
     # accepts our FileDescriptorProto, but it is not important what that file
     # name is actually set to.
-    proto_name = str(uuid.uuid4())
+    proto_name = binascii.hexlify(os.urandom(16)).decode('ascii')
 
     if package:
       file_descriptor_proto.name = os.path.join(package.replace('.', '/'),

python/google/protobuf/descriptor_database.py

@@ -32,6 +32,8 @@
 
 __author__ = 'matthewtoia@google.com (Matt Toia)'
 
+import warnings
+
 
 class Error(Exception):
   pass
@@ -64,21 +66,20 @@ class DescriptorDatabase(object):
     elif self._file_desc_protos_by_file[proto_name] != file_desc_proto:
       raise DescriptorDatabaseConflictingDefinitionError(
           '%s already added, but with different descriptor.' % proto_name)
+    else:
+      return
 
     # Add all the top-level descriptors to the index.
     package = file_desc_proto.package
     for message in file_desc_proto.message_type:
-      self._file_desc_protos_by_symbol.update(
-          (name, file_desc_proto) for name in _ExtractSymbols(message, package))
+      for name in _ExtractSymbols(message, package):
+        self._AddSymbol(name, file_desc_proto)
     for enum in file_desc_proto.enum_type:
-      self._file_desc_protos_by_symbol[
-          '.'.join((package, enum.name))] = file_desc_proto
+      self._AddSymbol(('.'.join((package, enum.name))), file_desc_proto)
     for extension in file_desc_proto.extension:
-      self._file_desc_protos_by_symbol[
-          '.'.join((package, extension.name))] = file_desc_proto
+      self._AddSymbol(('.'.join((package, extension.name))), file_desc_proto)
     for service in file_desc_proto.service:
-      self._file_desc_protos_by_symbol[
-          '.'.join((package, service.name))] = file_desc_proto
+      self._AddSymbol(('.'.join((package, service.name))), file_desc_proto)
 
   def FindFileByName(self, name):
     """Finds the file descriptor proto by file name.
@@ -132,6 +133,15 @@ class DescriptorDatabase(object):
       top_level, _, _ = symbol.rpartition('.')
       return self._file_desc_protos_by_symbol[top_level]
 
+  def _AddSymbol(self, name, file_desc_proto):
+    if name in self._file_desc_protos_by_symbol:
+      warn_msg = ('Conflict register for file "' + file_desc_proto.name +
+                  '": ' + name +
+                  ' is already defined in file "' +
+                  self._file_desc_protos_by_symbol[name].name + '"')
+      warnings.warn(warn_msg, RuntimeWarning)
+    self._file_desc_protos_by_symbol[name] = file_desc_proto
+
 
 def _ExtractSymbols(desc_proto, package):
   """Pulls out all the symbols from a descriptor proto.

python/google/protobuf/descriptor_pool.py

@@ -58,6 +58,7 @@ directly instead of this class.
 __author__ = 'matthewtoia@google.com (Matt Toia)'
 
 import collections
+import warnings
 
 from google.protobuf import descriptor
 from google.protobuf import descriptor_database
@@ -136,6 +137,29 @@ class DescriptorPool(object):
     self._extensions_by_name = collections.defaultdict(dict)
     self._extensions_by_number = collections.defaultdict(dict)
 
+  def _CheckConflictRegister(self, desc):
+    """Check if the descriptor name conflicts with another of the same name.
+
+    Args:
+      desc: Descriptor of a message, enum, service or extension.
+    """
+    desc_name = desc.full_name
+    for register, descriptor_type in [
+        (self._descriptors, descriptor.Descriptor),
+        (self._enum_descriptors, descriptor.EnumDescriptor),
+        (self._service_descriptors, descriptor.ServiceDescriptor),
+        (self._toplevel_extensions, descriptor.FieldDescriptor)]:
+      if desc_name in register:
+        file_name = register[desc_name].file.name
+        if not isinstance(desc, descriptor_type) or (
+            file_name != desc.file.name):
+          warn_msg = ('Conflict register for file "' + desc.file.name +
+                      '": ' + desc_name +
+                      ' is already defined in file "' +
+                      file_name + '"')
+          warnings.warn(warn_msg, RuntimeWarning)
+        return
+
   def Add(self, file_desc_proto):
     """Adds the FileDescriptorProto and its types to this pool.
 
@@ -172,6 +196,8 @@ class DescriptorPool(object):
     if not isinstance(desc, descriptor.Descriptor):
       raise TypeError('Expected instance of descriptor.Descriptor.')
 
+    self._CheckConflictRegister(desc)
+
     self._descriptors[desc.full_name] = desc
     self._AddFileDescriptor(desc.file)
 
@@ -187,6 +213,7 @@ class DescriptorPool(object):
     if not isinstance(enum_desc, descriptor.EnumDescriptor):
       raise TypeError('Expected instance of descriptor.EnumDescriptor.')
 
+    self._CheckConflictRegister(enum_desc)
     self._enum_descriptors[enum_desc.full_name] = enum_desc
     self._AddFileDescriptor(enum_desc.file)
 
@@ -200,6 +227,7 @@ class DescriptorPool(object):
     if not isinstance(service_desc, descriptor.ServiceDescriptor):
       raise TypeError('Expected instance of descriptor.ServiceDescriptor.')
 
+    self._CheckConflictRegister(service_desc)
     self._service_descriptors[service_desc.full_name] = service_desc
 
   def AddExtensionDescriptor(self, extension):
@@ -219,6 +247,7 @@ class DescriptorPool(object):
       raise TypeError('Expected an extension descriptor.')
 
     if extension.extension_scope is None:
+      self._CheckConflictRegister(extension)
       self._toplevel_extensions[extension.full_name] = extension
 
     try:
@@ -689,6 +718,7 @@ class DescriptorPool(object):
         fields[field_index].containing_oneof = oneofs[oneof_index]
 
     scope[_PrefixWithDot(desc_name)] = desc
+    self._CheckConflictRegister(desc)
     self._descriptors[desc_name] = desc
     return desc
 
@@ -727,6 +757,7 @@ class DescriptorPool(object):
                                      containing_type=containing_type,
                                      options=_OptionsOrNone(enum_proto))
     scope['.%s' % enum_name] = desc
+    self._CheckConflictRegister(desc)
     self._enum_descriptors[enum_name] = desc
     return desc
 
@@ -923,6 +954,7 @@ class DescriptorPool(object):
                                         methods=methods,
                                         options=_OptionsOrNone(service_proto),
                                         file=file_desc)
+    self._CheckConflictRegister(desc)
     self._service_descriptors[service_name] = desc
     return desc
 

python/google/protobuf/internal/descriptor_database_test.py

@@ -38,6 +38,7 @@ try:
   import unittest2 as unittest  #PY26
 except ImportError:
   import unittest
+import warnings
 
 from google.protobuf import unittest_pb2
 from google.protobuf import descriptor_pb2
@@ -98,6 +99,26 @@ class DescriptorDatabaseTest(unittest.TestCase):
                       db.FindFileContainingSymbol,
                       'protobuf_unittest.NoneMessage')
 
+  def testConflictRegister(self):
+    db = descriptor_database.DescriptorDatabase()
+    unittest_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb)
+    db.Add(unittest_fd)
+    conflict_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb)
+    conflict_fd.name = 'other_file'
+    with warnings.catch_warnings(record=True) as w:
+      # Cause all warnings to always be triggered.
+      warnings.simplefilter('always')
+      db.Add(conflict_fd)
+      self.assertTrue(len(w))
+      self.assertIs(w[0].category, RuntimeWarning)
+      self.assertIn('Conflict register for file "other_file": ',
+                    str(w[0].message))
+      self.assertIn('already defined in file '
+                    '"google/protobuf/unittest.proto"',
+                    str(w[0].message))
+
 
 if __name__ == '__main__':
   unittest.main()

python/google/protobuf/internal/descriptor_pool_test.py

@@ -34,8 +34,10 @@
 
 __author__ = 'matthewtoia@google.com (Matt Toia)'
 
+import copy
 import os
 import sys
+import warnings
 
 try:
   import unittest2 as unittest  #PY26
@@ -119,6 +121,14 @@ class DescriptorPoolTestBase(object):
     self.assertEqual('google/protobuf/unittest.proto',
                      file_desc5.name)
 
+    # Tests the generated pool.
+    assert descriptor_pool.Default().FindFileContainingSymbol(
+        'google.protobuf.python.internal.Factory2Message.one_more_field')
+    assert descriptor_pool.Default().FindFileContainingSymbol(
+        'google.protobuf.python.internal.another_field')
+    assert descriptor_pool.Default().FindFileContainingSymbol(
+        'protobuf_unittest.TestService')
+
   def testFindFileContainingSymbolFailure(self):
     with self.assertRaises(KeyError):
       self.pool.FindFileContainingSymbol('Does not exist')
@@ -492,6 +502,52 @@ class DescriptorPoolTestBase(object):
     TEST1_FILE.CheckFile(self, self.pool)
     TEST2_FILE.CheckFile(self, self.pool)
 
+  def testConflictRegister(self):
+    if isinstance(self, SecondaryDescriptorFromDescriptorDB):
+      if api_implementation.Type() == 'cpp':
+        # Cpp extension cannot call Add on a DescriptorPool
+        # that uses a DescriptorDatabase.
+        # TODO(jieluo): Fix python and cpp extension diff.
+        return
+    unittest_fd = descriptor_pb2.FileDescriptorProto.FromString(
+        unittest_pb2.DESCRIPTOR.serialized_pb)
+    conflict_fd = copy.deepcopy(unittest_fd)
+    conflict_fd.name = 'other_file'
+    if api_implementation.Type() == 'cpp':
+      try:
+        self.pool.Add(unittest_fd)
+        self.pool.Add(conflict_fd)
+      except TypeError:
+        pass
+    else:
+      with warnings.catch_warnings(record=True) as w:
+        # Cause all warnings to always be triggered.
+        warnings.simplefilter('always')
+        pool = copy.deepcopy(self.pool)
+        # No warnings to add the same descriptors.
+        file_descriptor = unittest_pb2.DESCRIPTOR
+        pool.AddDescriptor(
+            file_descriptor.message_types_by_name['TestAllTypes'])
+        pool.AddEnumDescriptor(
+            file_descriptor.enum_types_by_name['ForeignEnum'])
+        pool.AddServiceDescriptor(
+            file_descriptor.services_by_name['TestService'])
+        pool.AddExtensionDescriptor(
+            file_descriptor.extensions_by_name['optional_int32_extension'])
+        self.assertEqual(len(w), 0)
+        # Check warnings for conflict descriptors with the same name.
+        pool.Add(unittest_fd)
+        pool.Add(conflict_fd)
+        pool.FindFileByName(unittest_fd.name)
+        pool.FindFileByName(conflict_fd.name)
+        self.assertTrue(len(w))
+        self.assertIs(w[0].category, RuntimeWarning)
+        self.assertIn('Conflict register for file "other_file": ',
+                      str(w[0].message))
+        self.assertIn('already defined in file '
+                      '"google/protobuf/unittest.proto"',
+                      str(w[0].message))
+
 
 class DefaultDescriptorPoolTest(DescriptorPoolTestBase, unittest.TestCase):
 

python/google/protobuf/internal/encoder.py

@@ -418,7 +418,8 @@ def _VarintBytes(value):
 def TagBytes(field_number, wire_type):
   """Encode the given tag and return the bytes.  Only called at startup."""
 
-  return six.binary_type( _VarintBytes(wire_format.PackTag(field_number, wire_type)) )
+  return six.binary_type(
+      _VarintBytes(wire_format.PackTag(field_number, wire_type)))
 
 # --------------------------------------------------------------------
 # As with sizers (see above), we have a number of common encoder

python/google/protobuf/internal/text_format_test.py

@@ -828,6 +828,10 @@ class Proto2Tests(TextFormatBase):
             '    }\n'
             '  }\n'
             '  [unknown_extension]: 5\n'
+            '  [unknown_extension_with_number_field] {\n'
+            '    1: "some_field"\n'
+            '    2: -0.451\n'
+            '  }\n'
             '}\n')
     text_format.Parse(text, message, allow_unknown_extension=True)
     golden = 'message_set {\n}\n'
@@ -894,7 +898,6 @@ class Proto2Tests(TextFormatBase):
     message = unittest_mset_pb2.TestMessageSetContainer()
     malformed = ('message_set {\n'
                  '  unknown_field: true\n'
-                 '  \n'  # Missing '>' here.
                  '}\n')
     six.assertRaisesRegex(self,
                           text_format.ParseError,
@@ -906,7 +909,7 @@ class Proto2Tests(TextFormatBase):
                           message,
                           allow_unknown_extension=True)
 
-    # Parse known extension correcty.
+    # Parse known extension correctly.
     message = unittest_mset_pb2.TestMessageSetContainer()
     text = ('message_set {\n'
             '  [protobuf_unittest.TestMessageSetExtension1] {\n'

python/google/protobuf/pyext/message.cc

@@ -605,18 +605,16 @@ void OutOfRangeError(PyObject* arg) {
 
 template<class RangeType, class ValueType>
 bool VerifyIntegerCastAndRange(PyObject* arg, ValueType value) {
-  if
-    GOOGLE_PREDICT_FALSE(value == -1 && PyErr_Occurred()) {
-      if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
-        // Replace it with the same ValueError as pure python protos instead of
-        // the default one.
-        PyErr_Clear();
-        OutOfRangeError(arg);
-      }  // Otherwise propagate existing error.
-      return false;
+  if (GOOGLE_PREDICT_FALSE(value == -1 && PyErr_Occurred())) {
+    if (PyErr_ExceptionMatches(PyExc_OverflowError)) {
+      // Replace it with the same ValueError as pure python protos instead of
+      // the default one.
+      PyErr_Clear();
+      OutOfRangeError(arg);
+    }  // Otherwise propagate existing error.
+    return false;
     }
-  if
-    GOOGLE_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value)) {
+    if (GOOGLE_PREDICT_FALSE(!IsValidNumericCast<RangeType>(value))) {
       OutOfRangeError(arg);
       return false;
     }
@@ -628,26 +626,22 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
   // The fast path.
 #if PY_MAJOR_VERSION < 3
   // For the typical case, offer a fast path.
-  if
-    GOOGLE_PREDICT_TRUE(PyInt_Check(arg)) {
-      long int_result = PyInt_AsLong(arg);
-      if
-        GOOGLE_PREDICT_TRUE(IsValidNumericCast<T>(int_result)) {
-          *value = static_cast<T>(int_result);
-          return true;
-        }
-      else {
-        OutOfRangeError(arg);
-        return false;
-      }
+  if (GOOGLE_PREDICT_TRUE(PyInt_Check(arg))) {
+    long int_result = PyInt_AsLong(arg);
+    if (GOOGLE_PREDICT_TRUE(IsValidNumericCast<T>(int_result))) {
+      *value = static_cast<T>(int_result);
+      return true;
+    } else {
+      OutOfRangeError(arg);
+      return false;
+    }
     }
 #endif
   // This effectively defines an integer as "an object that can be cast as
   // an integer and can be used as an ordinal number".
   // This definition includes everything that implements numbers.Integral
   // and shouldn't cast the net too wide.
-  if
-    GOOGLE_PREDICT_FALSE(!PyIndex_Check(arg)) {
+    if (GOOGLE_PREDICT_FALSE(!PyIndex_Check(arg))) {
       FormatTypeError(arg, "int, long");
       return false;
     }
@@ -664,10 +658,9 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Unlike PyLong_AsLongLong, PyLong_AsUnsignedLongLong is very
       // picky about the exact type.
       PyObject* casted = PyNumber_Long(arg);
-      if
-        GOOGLE_PREDICT_FALSE(casted == NULL) {
-          // Propagate existing error.
-          return false;
+      if (GOOGLE_PREDICT_FALSE(casted == NULL)) {
+        // Propagate existing error.
+        return false;
         }
       ulong_result = PyLong_AsUnsignedLongLong(casted);
       Py_DECREF(casted);
@@ -690,10 +683,9 @@ bool CheckAndGetInteger(PyObject* arg, T* value) {
       // Valid subclasses of numbers.Integral should have a __long__() method
       // so fall back to that.
       PyObject* casted = PyNumber_Long(arg);
-      if
-        GOOGLE_PREDICT_FALSE(casted == NULL) {
-          // Propagate existing error.
-          return false;
+      if (GOOGLE_PREDICT_FALSE(casted == NULL)) {
+        // Propagate existing error.
+        return false;
         }
       long_result = PyLong_AsLongLong(casted);
       Py_DECREF(casted);
@@ -717,10 +709,9 @@ template bool CheckAndGetInteger<uint64>(PyObject*, uint64*);
 
 bool CheckAndGetDouble(PyObject* arg, double* value) {
   *value = PyFloat_AsDouble(arg);
-  if
-    GOOGLE_PREDICT_FALSE(*value == -1 && PyErr_Occurred()) {
-      FormatTypeError(arg, "int, long, float");
-      return false;
+  if (GOOGLE_PREDICT_FALSE(*value == -1 && PyErr_Occurred())) {
+    FormatTypeError(arg, "int, long, float");
+    return false;
     }
   return true;
 }
@@ -1187,7 +1178,7 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
       continue;
     }
     if (descriptor->is_map()) {
-      ScopedPyObjectPtr map(GetAttr(self, name));
+      ScopedPyObjectPtr map(GetAttr(reinterpret_cast<PyObject*>(self), name));
       const FieldDescriptor* value_descriptor =
           descriptor->message_type()->FindFieldByName("value");
       if (value_descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
@@ -1215,7 +1206,8 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
         }
       }
     } else if (descriptor->label() == FieldDescriptor::LABEL_REPEATED) {
-      ScopedPyObjectPtr container(GetAttr(self, name));
+      ScopedPyObjectPtr container(
+          GetAttr(reinterpret_cast<PyObject*>(self), name));
       if (container == NULL) {
         return -1;
       }
@@ -1282,7 +1274,8 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
         }
       }
     } else if (descriptor->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-      ScopedPyObjectPtr message(GetAttr(self, name));
+      ScopedPyObjectPtr message(
+          GetAttr(reinterpret_cast<PyObject*>(self), name));
       if (message == NULL) {
         return -1;
       }
@@ -1307,8 +1300,8 @@ int InitAttributes(CMessage* self, PyObject* args, PyObject* kwargs) {
           return -1;
         }
       }
-      if (SetAttr(self, name, (new_val.get() == NULL) ? value : new_val.get()) <
-          0) {
+      if (SetAttr(reinterpret_cast<PyObject*>(self), name,
+                  (new_val.get() == NULL) ? value : new_val.get()) < 0) {
         return -1;
       }
     }
@@ -2211,7 +2204,8 @@ static PyObject* ListFields(CMessage* self) {
         return NULL;
       }
 
-      PyObject* field_value = GetAttr(self, py_field_name.get());
+      PyObject* field_value =
+          GetAttr(reinterpret_cast<PyObject*>(self), py_field_name.get());
       if (field_value == NULL) {
         PyErr_SetObject(PyExc_ValueError, py_field_name.get());
         return NULL;
@@ -2707,7 +2701,8 @@ static bool SetCompositeField(
   return PyDict_SetItem(self->composite_fields, name, value) == 0;
 }
 
-PyObject* GetAttr(CMessage* self, PyObject* name) {
+PyObject* GetAttr(PyObject* pself, PyObject* name) {
+  CMessage* self = reinterpret_cast<CMessage*>(pself);
   PyObject* value = self->composite_fields ?
       PyDict_GetItem(self->composite_fields, name) : NULL;
   if (value != NULL) {
@@ -2785,7 +2780,8 @@ PyObject* GetAttr(CMessage* self, PyObject* name) {
   return InternalGetScalar(self->message, field_descriptor);
 }
 
-int SetAttr(CMessage* self, PyObject* name, PyObject* value) {
+int SetAttr(PyObject* pself, PyObject* name, PyObject* value) {
+  CMessage* self = reinterpret_cast<CMessage*>(pself);
   if (self->composite_fields && PyDict_Contains(self->composite_fields, name)) {
     PyErr_SetString(PyExc_TypeError, "Can't set composite field");
     return -1;
@@ -2837,8 +2833,8 @@ PyTypeObject CMessage_Type = {
   PyObject_HashNotImplemented,         //  tp_hash
   0,                                   //  tp_call
   (reprfunc)cmessage::ToStr,           //  tp_str
-  (getattrofunc)cmessage::GetAttr,     //  tp_getattro
-  (setattrofunc)cmessage::SetAttr,     //  tp_setattro
+  cmessage::GetAttr,                   //  tp_getattro
+  cmessage::SetAttr,                   //  tp_setattro
   0,                                   //  tp_as_buffer
   Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,  //  tp_flags
   "A ProtocolMessage",                 //  tp_doc

python/google/protobuf/pyext/message.h

@@ -240,15 +240,15 @@ PyObject* MergeFrom(CMessage* self, PyObject* arg);
 // has been registered with the same field number on this class.
 PyObject* RegisterExtension(PyObject* cls, PyObject* extension_handle);
 
-// Retrieves an attribute named 'name' from CMessage 'self'. Returns
-// the attribute value on success, or NULL on failure.
+// Retrieves an attribute named 'name' from 'self', which is interpreted as a
+// CMessage. Returns the attribute value on success, or null on failure.
 //
 // Returns a new reference.
-PyObject* GetAttr(CMessage* self, PyObject* name);
+PyObject* GetAttr(PyObject* self, PyObject* name);
 
-// Set the value of the attribute named 'name', for CMessage 'self',
-// to the value 'value'. Returns -1 on failure.
-int SetAttr(CMessage* self, PyObject* name, PyObject* value);
+// Set the value of the attribute named 'name', for 'self', which is interpreted
+// as a CMessage, to the value 'value'. Returns -1 on failure.
+int SetAttr(PyObject* self, PyObject* name, PyObject* value);
 
 PyObject* FindInitializationErrors(CMessage* self);
 

python/google/protobuf/text_format.py

@@ -938,7 +938,7 @@ def _SkipField(tokenizer):
       tokenizer.ConsumeIdentifier()
     tokenizer.Consume(']')
   else:
-    tokenizer.ConsumeIdentifier()
+    tokenizer.ConsumeIdentifierOrNumber()
 
   _SkipFieldContents(tokenizer)
 

src/Makefile.am

@@ -118,6 +118,7 @@ nobase_include_HEADERS =                                         \
   google/protobuf/generated_message_table_driven.h               \
   google/protobuf/generated_message_util.h                       \
   google/protobuf/has_bits.h                                     \
+  google/protobuf/implicit_weak_message.h                        \
   google/protobuf/map_entry.h                                    \
   google/protobuf/map_entry_lite.h                               \
   google/protobuf/map_field.h                                    \
@@ -222,6 +223,7 @@ libprotobuf_lite_la_SOURCES =                                  \
   google/protobuf/generated_message_util.cc                    \
   google/protobuf/generated_message_table_driven_lite.h        \
   google/protobuf/generated_message_table_driven_lite.cc       \
+  google/protobuf/implicit_weak_message.cc                     \
   google/protobuf/message_lite.cc                              \
   google/protobuf/repeated_field.cc                            \
   google/protobuf/wire_format_lite.cc                          \

src/google/protobuf/any.pb.cc

@@ -196,13 +196,6 @@ const Any& Any::default_instance() {
   return *internal_default_instance();
 }
 
-Any* Any::New(::google::protobuf::Arena* arena) const {
-  Any* n = new Any;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
 
 void Any::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Any)
@@ -436,5 +429,12 @@ void Any::InternalSwap(Any* other) {
 // @@protoc_insertion_point(namespace_scope)
 }  // namespace protobuf
 }  // namespace google
+namespace google {
+namespace protobuf {
+template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::google::protobuf::Any* Arena::Create< ::google::protobuf::Any >(Arena* arena) {
+  return Arena::CreateInternal< ::google::protobuf::Any >(arena);
+}
+}  // namespace protobuf
+}  // namespace google
 
 // @@protoc_insertion_point(global_scope)

src/google/protobuf/any.pb.h

@@ -58,6 +58,11 @@ LIBPROTOBUF_EXPORT extern AnyDefaultTypeInternal _Any_default_instance_;
 }  // namespace google
 namespace google {
 namespace protobuf {
+template<> LIBPROTOBUF_EXPORT ::google::protobuf::Any* Arena::Create< ::google::protobuf::Any>(Arena*);
+}  // namespace protobuf
+}  // namespace google
+namespace google {
+namespace protobuf {
 
 // ===================================================================
 
@@ -115,9 +120,13 @@ class LIBPROTOBUF_EXPORT Any : public ::google::protobuf::Message /* @@protoc_in
 
   // implements Message ----------------------------------------------
 
-  inline Any* New() const PROTOBUF_FINAL { return New(NULL); }
+  inline Any* New() const PROTOBUF_FINAL {
+    return ::google::protobuf::Arena::Create<Any>(NULL);
+  }
 
-  Any* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL;
+  Any* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL {
+    return ::google::protobuf::Arena::Create<Any>(arena);
+  }
   void CopyFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
   void MergeFrom(const ::google::protobuf::Message& from) PROTOBUF_FINAL;
   void CopyFrom(const Any& from);

src/google/protobuf/any.proto

@@ -120,17 +120,18 @@ option objc_class_prefix = "GPB";
 //     }
 //
 message Any {
-  // A URL/resource name whose content describes the type of the
-  // serialized protocol buffer message.
+  // A URL/resource name that uniquely identifies the type of the serialized
+  // protocol buffer message. The last segment of the URL's path must represent
+  // the fully qualified name of the type (as in
+  // `path/google.protobuf.Duration`). The name should be in a canonical form
+  // (e.g., leading "." is not accepted).
   //
-  // For URLs which use the scheme `http`, `https`, or no scheme, the
-  // following restrictions and interpretations apply:
+  // In practice, teams usually precompile into the binary all types that they
+  // expect it to use in the context of Any. However, for URLs which use the
+  // scheme `http`, `https`, or no scheme, one can optionally set up a type
+  // server that maps type URLs to message definitions as follows:
   //
   // * If no scheme is provided, `https` is assumed.
-  // * The last segment of the URL's path must represent the fully
-  //   qualified name of the type (as in `path/google.protobuf.Duration`).
-  //   The name should be in a canonical form (e.g., leading "." is
-  //   not accepted).
   // * An HTTP GET on the URL must yield a [google.protobuf.Type][]
   //   value in binary format, or produce an error.
   // * Applications are allowed to cache lookup results based on the
@@ -139,6 +140,10 @@ message Any {
   //   on changes to types. (Use versioned type names to manage
   //   breaking changes.)
   //
+  // Note: this functionality is not currently available in the official
+  // protobuf release, and it is not used for type URLs beginning with
+  // type.googleapis.com.
+  //
   // Schemes other than `http`, `https` (or the empty scheme) might be
   // used with implementation specific semantics.
   //

src/google/protobuf/api.pb.cc

@@ -311,13 +311,6 @@ const Api& Api::default_instance() {
   return *internal_default_instance();
 }
 
-Api* Api::New(::google::protobuf::Arena* arena) const {
-  Api* n = new Api;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
 
 void Api::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Api)
@@ -368,7 +361,8 @@ bool Api::MergePartialFromCodedStream(
       case 2: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_methods()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
+                input, add_methods()));
         } else {
           goto handle_unusual;
         }
@@ -379,7 +373,8 @@ bool Api::MergePartialFromCodedStream(
       case 3: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
+                input, add_options()));
         } else {
           goto handle_unusual;
         }
@@ -418,7 +413,8 @@ bool Api::MergePartialFromCodedStream(
       case 6: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_mixins()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
+                input, add_mixins()));
         } else {
           goto handle_unusual;
         }
@@ -480,14 +476,18 @@ void Api::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->methods_size()); i < n; i++) {
     ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
-      2, this->methods(static_cast<int>(i)), output);
+      2,
+      this->methods(static_cast<int>(i)),
+      output);
   }
 
   // repeated .google.protobuf.Option options = 3;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->options_size()); i < n; i++) {
     ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
-      3, this->options(static_cast<int>(i)), output);
+      3,
+      this->options(static_cast<int>(i)),
+      output);
   }
 
   // string version = 4;
@@ -503,14 +503,16 @@ void Api::SerializeWithCachedSizes(
   // .google.protobuf.SourceContext source_context = 5;
   if (this->has_source_context()) {
     ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
-      5, *this->source_context_, output);
+      5, *source_context_, output);
   }
 
   // repeated .google.protobuf.Mixin mixins = 6;
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->mixins_size()); i < n; i++) {
     ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
-      6, this->mixins(static_cast<int>(i)), output);
+      6,
+      this->mixins(static_cast<int>(i)),
+      output);
   }
 
   // .google.protobuf.Syntax syntax = 7;
@@ -575,7 +577,7 @@ void Api::SerializeWithCachedSizes(
   if (this->has_source_context()) {
     target = ::google::protobuf::internal::WireFormatLite::
       InternalWriteMessageToArray(
-        5, *this->source_context_, deterministic, target);
+        5, *source_context_, deterministic, target);
   }
 
   // repeated .google.protobuf.Mixin mixins = 6;
@@ -660,7 +662,7 @@ size_t Api::ByteSizeLong() const {
   if (this->has_source_context()) {
     total_size += 1 +
       ::google::protobuf::internal::WireFormatLite::MessageSize(
-        *this->source_context_);
+        *source_context_);
   }
 
   // .google.protobuf.Syntax syntax = 7;
@@ -741,9 +743,9 @@ void Api::Swap(Api* other) {
 }
 void Api::InternalSwap(Api* other) {
   using std::swap;
-  methods_.InternalSwap(&other->methods_);
-  options_.InternalSwap(&other->options_);
-  mixins_.InternalSwap(&other->mixins_);
+  CastToBase(&methods_)->InternalSwap(CastToBase(&other->methods_));
+  CastToBase(&options_)->InternalSwap(CastToBase(&other->options_));
+  CastToBase(&mixins_)->InternalSwap(CastToBase(&other->mixins_));
   name_.Swap(&other->name_);
   version_.Swap(&other->version_);
   swap(source_context_, other->source_context_);
@@ -843,13 +845,6 @@ const Method& Method::default_instance() {
   return *internal_default_instance();
 }
 
-Method* Method::New(::google::protobuf::Arena* arena) const {
-  Method* n = new Method;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
 
 void Method::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Method)
@@ -957,7 +952,8 @@ bool Method::MergePartialFromCodedStream(
       case 6: {
         if (static_cast< ::google::protobuf::uint8>(tag) ==
             static_cast< ::google::protobuf::uint8>(50u /* 50 & 0xFF */)) {
-          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(input, add_options()));
+          DO_(::google::protobuf::internal::WireFormatLite::ReadMessage(
+                input, add_options()));
         } else {
           goto handle_unusual;
         }
@@ -1049,7 +1045,9 @@ void Method::SerializeWithCachedSizes(
   for (unsigned int i = 0,
       n = static_cast<unsigned int>(this->options_size()); i < n; i++) {
     ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray(
-      6, this->options(static_cast<int>(i)), output);
+      6,
+      this->options(static_cast<int>(i)),
+      output);
   }
 
   // .google.protobuf.Syntax syntax = 7;
@@ -1271,7 +1269,7 @@ void Method::Swap(Method* other) {
 }
 void Method::InternalSwap(Method* other) {
   using std::swap;
-  options_.InternalSwap(&other->options_);
+  CastToBase(&options_)->InternalSwap(CastToBase(&other->options_));
   name_.Swap(&other->name_);
   request_type_url_.Swap(&other->request_type_url_);
   response_type_url_.Swap(&other->response_type_url_);
@@ -1352,13 +1350,6 @@ const Mixin& Mixin::default_instance() {
   return *internal_default_instance();
 }
 
-Mixin* Mixin::New(::google::protobuf::Arena* arena) const {
-  Mixin* n = new Mixin;
-  if (arena != NULL) {
-    arena->Own(n);
-  }
-  return n;
-}
 
 void Mixin::Clear() {
 // @@protoc_insertion_point(message_clear_start:google.protobuf.Mixin)
@@ -1604,5 +1595,18 @@ void Mixin::InternalSwap(Mixin* other) {
 // @@protoc_insertion_point(namespace_scope)
 }  // namespace protobuf
 }  // namespace google
+namespace google {
+namespace protobuf {
+template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::google::protobuf::Api* Arena::Create< ::google::protobuf::Api >(Arena* arena) {
+  return Arena::CreateInternal< ::google::protobuf::Api >(arena);
+}
+template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::google::protobuf::Method* Arena::Create< ::google::protobuf::Method >(Arena* arena) {
+  return Arena::CreateInternal< ::google::protobuf::Method >(arena);
+}
+template<> GOOGLE_PROTOBUF_ATTRIBUTE_NOINLINE ::google::protobuf::Mixin* Arena::Create< ::google::protobuf::Mixin >(Arena* arena) {
+  return Arena::CreateInternal< ::google::protobuf::Mixin >(arena);
+}
+}  // namespace protobuf
+}  // namespace google
 
 // @@protoc_insertion_point(global_scope)

src/google/protobuf/arena.h

@@ -56,6 +56,21 @@ using type_info = ::type_info;
 
 namespace google {
 namespace protobuf {
+struct ArenaOptions;
+}  // namespace protobuf
+
+namespace quality_webanswers {
+
+void TempPrivateWorkAround(::google::protobuf::ArenaOptions* arena_options);
+
+}  // namespace quality_webanswers
+
+namespace protobuf {
+namespace arena_metrics {
+
+void EnableArenaMetrics(::google::protobuf::ArenaOptions* options);
+
+}  // namespace arena_metrics
 
 class Arena;       // defined below
 class Message;     // message.h
@@ -117,6 +132,20 @@ struct ArenaOptions {
   // from the arena. By default, it contains a ptr to a wrapper function that
   // calls free.
   void (*block_dealloc)(void*, size_t);
+
+  ArenaOptions()
+      : start_block_size(kDefaultStartBlockSize),
+        max_block_size(kDefaultMaxBlockSize),
+        initial_block(NULL),
+        initial_block_size(0),
+        block_alloc(&::operator new),
+        block_dealloc(&internal::arena_free),
+        on_arena_init(NULL),
+        on_arena_reset(NULL),
+        on_arena_destruction(NULL),
+        on_arena_allocation(NULL) {}
+
+ private:
   // Hooks for adding external functionality such as user-specific metrics
   // collection, specific debugging abilities, etc.
   // Init hook may return a pointer to a cookie to be stored in the arena.
@@ -138,23 +167,15 @@ struct ArenaOptions {
   void (*on_arena_allocation)(const std::type_info* allocated_type,
       uint64 alloc_size, void* cookie);
 
-  ArenaOptions()
-      : start_block_size(kDefaultStartBlockSize),
-        max_block_size(kDefaultMaxBlockSize),
-        initial_block(NULL),
-        initial_block_size(0),
-        block_alloc(&::operator new),
-        block_dealloc(&internal::arena_free),
-        on_arena_init(NULL),
-        on_arena_reset(NULL),
-        on_arena_destruction(NULL),
-        on_arena_allocation(NULL) {}
-
- private:
   // Constants define default starting block size and max block size for
   // arena allocator behavior -- see descriptions above.
   static const size_t kDefaultStartBlockSize = 256;
   static const size_t kDefaultMaxBlockSize   = 8192;
+
+  friend void ::google::protobuf::arena_metrics::EnableArenaMetrics(ArenaOptions*);
+  friend void quality_webanswers::TempPrivateWorkAround(ArenaOptions*);
+  friend class Arena;
+  friend class ArenaOptionsTestFriend;
 };
 
 // Support for non-RTTI environments. (The metrics hooks API uses type
@@ -229,14 +250,15 @@ class LIBPROTOBUF_EXPORT Arena {
   // WARNING: if you allocate multiple objects, it is difficult to guarantee
   // that a series of allocations will fit in the initial block, especially if
   // Arena changes its alignment guarantees in the future!
-  static const size_t kBlockOverhead = internal::ArenaImpl::kHeaderSize;
+  static const size_t kBlockOverhead = internal::ArenaImpl::kBlockHeaderSize +
+                                       internal::ArenaImpl::kSerialArenaSize;
 
   // Default constructor with sensible default options, tuned for average
   // use-cases.
   Arena() : impl_(ArenaOptions()) { Init(ArenaOptions()); }
 
   ~Arena() {
-    if (on_arena_reset_ != NULL || on_arena_destruction_ != NULL) {
+    if (hooks_cookie_) {
       CallDestructorHooks();
     }
   }
@@ -277,6 +299,7 @@ class LIBPROTOBUF_EXPORT Arena {
     }
   }
 #endif
+
   template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMessage(::google::protobuf::Arena* arena) {
 #if LANG_CXX11
@@ -355,6 +378,7 @@ class LIBPROTOBUF_EXPORT Arena {
     }
   }
 #endif
+
   template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* Create(::google::protobuf::Arena* arena) {
     if (arena == NULL) {
@@ -522,6 +546,7 @@ class LIBPROTOBUF_EXPORT Arena {
   //
   // Combines SpaceAllocated and SpaceUsed. Returns a pair of
   // <space_allocated, space_used>.
+  PROTOBUF_RUNTIME_DEPRECATED("Please use SpaceAllocated() and SpaceUsed()")
   std::pair<uint64, uint64> SpaceAllocatedAndUsed() const {
     return std::make_pair(SpaceAllocated(), SpaceUsed());
   }
@@ -637,6 +662,29 @@ class LIBPROTOBUF_EXPORT Arena {
   struct is_arena_constructable : InternalHelper<T>::is_arena_constructable {};
 
  private:
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateMessageInternal(::google::protobuf::Arena* arena) {
+#if LANG_CXX11
+    static_assert(
+        InternalHelper<T>::is_arena_constructable::value,
+        "CreateMessage can only construct types that are ArenaConstructable");
+#endif
+    if (arena == NULL) {
+      return new T;
+    } else {
+      return arena->CreateMessageInternal<T>();
+    }
+  }
+
+  template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
+  static T* CreateInternal(::google::protobuf::Arena* arena) {
+    if (arena == NULL) {
+      return new T();
+    } else {
+      return arena->CreateInternal<T>(google::protobuf::internal::has_trivial_destructor<T>::value);
+    }
+  }
+
   void CallDestructorHooks();
   void OnArenaAllocation(const std::type_info* allocated_type, size_t n) const;
   inline void AllocHook(const std::type_info* allocated_type, size_t n) const {
@@ -668,12 +716,12 @@ class LIBPROTOBUF_EXPORT Arena {
   // fields, since they are designed to work in all mode combinations.
   template <typename Msg> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static Msg* CreateMaybeMessage(Arena* arena, google::protobuf::internal::true_type) {
-    return CreateMessage<Msg>(arena);
+    return CreateMessageInternal<Msg>(arena);
   }
 
   template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
   static T* CreateMaybeMessage(Arena* arena, google::protobuf::internal::false_type) {
-    return Create<T>(arena);
+    return CreateInternal<T>(arena);
   }
 
   template <typename T> GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
@@ -907,7 +955,6 @@ class LIBPROTOBUF_EXPORT Arena {
 
   internal::ArenaImpl impl_;
 
-  void* (*on_arena_init_)(Arena* arena);
   void (*on_arena_allocation_)(const std::type_info* allocated_type,
                                uint64 alloc_size, void* cookie);
   void (*on_arena_reset_)(Arena* arena, void* cookie, uint64 space_used);

src/google/protobuf/arena_impl.h

@@ -40,10 +40,13 @@
 #include <google/protobuf/stubs/common.h>
 #include <google/protobuf/stubs/logging.h>
 #include <google/protobuf/stubs/mutex.h>
-#include <google/protobuf/stubs/type_traits.h>
 
 #include <google/protobuf/stubs/port.h>
 
+#ifdef ADDRESS_SANITIZER
+#include <sanitizer/asan_interface.h>
+#endif  // ADDRESS_SANITIZER
+
 namespace google {
 
 namespace protobuf {
@@ -112,6 +115,10 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
   void AddCleanup(void* elem, void (*cleanup)(void*));
 
  private:
+  void* AllocateAlignedFallback(size_t n);
+  void* AllocateAlignedAndAddCleanupFallback(size_t n, void (*cleanup)(void*));
+  void AddCleanupFallback(void* elem, void (*cleanup)(void*));
+
   // Node contains the ptr of the object to be cleaned up and the associated
   // cleanup function ptr.
   struct CleanupNode {
@@ -124,34 +131,107 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
     static size_t SizeOf(size_t i) {
       return sizeof(CleanupChunk) + (sizeof(CleanupNode) * (i - 1));
     }
-    size_t len;            // Number of elements currently present.
     size_t size;           // Total elements in the list.
     CleanupChunk* next;    // Next node in the list.
     CleanupNode nodes[1];  // True length is |size|.
   };
 
-  struct Block;
+  class Block;
+
+  // A thread-unsafe Arena that can only be used within its owning thread.
+  class LIBPROTOBUF_EXPORT SerialArena {
+   public:
+    // The allocate/free methods here are a little strange, since SerialArena is
+    // allocated inside a Block which it also manages.  This is to avoid doing
+    // an extra allocation for the SerialArena itself.
+
+    // Creates a new SerialArena inside Block* and returns it.
+    static SerialArena* New(Block* b, void* owner, ArenaImpl* arena);
+
+    // Destroys this SerialArena, freeing all blocks with the given dealloc
+    // function, except any block equal to |initial_block|.
+    static uint64 Free(SerialArena* serial, Block* initial_block,
+                       void (*block_dealloc)(void*, size_t));
+
+    void CleanupList();
+    uint64 SpaceUsed() const;
+
+    void* AllocateAligned(size_t n) {
+      GOOGLE_DCHECK_EQ(internal::AlignUpTo8(n), n);  // Must be already aligned.
+      GOOGLE_DCHECK_GE(limit_, ptr_);
+      if (GOOGLE_PREDICT_FALSE(static_cast<size_t>(limit_ - ptr_) < n)) {
+        return AllocateAlignedFallback(n);
+      }
+      void* ret = ptr_;
+      ptr_ += n;
+#ifdef ADDRESS_SANITIZER
+      ASAN_UNPOISON_MEMORY_REGION(ret, n);
+#endif  // ADDRESS_SANITIZER
+      return ret;
+    }
+
+    void AddCleanup(void* elem, void (*cleanup)(void*)) {
+      if (GOOGLE_PREDICT_FALSE(cleanup_ptr_ == cleanup_limit_)) {
+        AddCleanupFallback(elem, cleanup);
+        return;
+      }
+      cleanup_ptr_->elem = elem;
+      cleanup_ptr_->cleanup = cleanup;
+      cleanup_ptr_++;
+    }
+
+    void* AllocateAlignedAndAddCleanup(size_t n, void (*cleanup)(void*)) {
+      void* ret = AllocateAligned(n);
+      AddCleanup(ret, cleanup);
+      return ret;
+    }
 
-  // Tracks per-thread info.  ThreadInfos are kept in a linked list.
-  struct ThreadInfo {
-    void *owner;             // &ThreadCache of this thread;
-    Block* head;             // Head of linked list of blocks.
-    CleanupChunk* cleanup;   // Head of cleanup list.
-    ThreadInfo* next;        // Next ThreadInfo in this linked list.
+    void* owner() const { return owner_; }
+    SerialArena* next() const { return next_; }
+    void set_next(SerialArena* next) { next_ = next; }
+
+   private:
+    void* AllocateAlignedFallback(size_t n);
+    void AddCleanupFallback(void* elem, void (*cleanup)(void*));
+    void CleanupListFallback();
+
+    ArenaImpl* arena_;        // Containing arena.
+    void* owner_;             // &ThreadCache of this thread;
+    Block* head_;             // Head of linked list of blocks.
+    CleanupChunk* cleanup_;   // Head of cleanup list.
+    SerialArena* next_;       // Next SerialArena in this linked list.
+
+    // Next pointer to allocate from.  Always 8-byte aligned.  Points inside
+    // head_ (and head_->pos will always be non-canonical).  We keep these
+    // here to reduce indirection.
+    char* ptr_;
+    char* limit_;
+
+    // Next CleanupList members to append to.  These point inside cleanup_.
+    CleanupNode* cleanup_ptr_;
+    CleanupNode* cleanup_limit_;
   };
 
   // Blocks are variable length malloc-ed objects.  The following structure
   // describes the common header for all blocks.
-  struct Block {
-    void* owner;              // &ThreadCache of thread that owns this block.
-    ThreadInfo* thread_info;  // ThreadInfo of thread that owns this block.
-    Block* next;              // Next block in arena (may have different owner)
-    // ((char*) &block) + pos is next available byte. It is always
-    // aligned at a multiple of 8 bytes.
-    size_t pos;
-    size_t size;  // total size of the block.
-    GOOGLE_PROTOBUF_ATTRIBUTE_ALWAYS_INLINE
-    size_t avail() const { return size - pos; }
+  class LIBPROTOBUF_EXPORT Block {
+   public:
+    Block(size_t size, Block* next);
+
+    char* Pointer(size_t n) {
+      GOOGLE_DCHECK(n <= size_);
+      return reinterpret_cast<char*>(this) + n;
+    }
+
+    Block* next() const { return next_; }
+    size_t pos() const { return pos_; }
+    size_t size() const { return size_; }
+    void set_pos(size_t pos) { pos_ = pos; }
+
+   private:
+    Block* next_;   // Next block for this thread.
+    size_t pos_;
+    size_t size_;
     // data follows
   };
 
@@ -160,13 +240,13 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
     // If we are using the ThreadLocalStorage class to store the ThreadCache,
     // then the ThreadCache's default constructor has to be responsible for
     // initializing it.
-    ThreadCache() : last_lifecycle_id_seen(-1), last_block_used_(NULL) {}
+    ThreadCache() : last_lifecycle_id_seen(-1), last_serial_arena(NULL) {}
 #endif
 
     // The ThreadCache is considered valid as long as this matches the
     // lifecycle_id of the arena being used.
     int64 last_lifecycle_id_seen;
-    Block* last_block_used_;
+    SerialArena* last_serial_arena;
   };
   static google::protobuf::internal::SequenceNumber lifecycle_id_generator_;
 #if defined(GOOGLE_PROTOBUF_NO_THREADLOCAL)
@@ -188,38 +268,30 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
   // Free all blocks and return the total space used which is the sums of sizes
   // of the all the allocated blocks.
   uint64 FreeBlocks();
-
-  void AddCleanupInBlock(Block* b, void* elem, void (*func)(void*));
-  CleanupChunk* ExpandCleanupList(CleanupChunk* cleanup, Block* b);
   // Delete or Destruct all objects owned by the arena.
   void CleanupList();
 
-  inline void CacheBlock(Block* block) {
-    thread_cache().last_block_used_ = block;
+  inline void CacheSerialArena(SerialArena* serial) {
+    thread_cache().last_serial_arena = serial;
     thread_cache().last_lifecycle_id_seen = lifecycle_id_;
     // TODO(haberman): evaluate whether we would gain efficiency by getting rid
     // of hint_.  It's the only write we do to ArenaImpl in the allocation path,
     // which will dirty the cache line.
-    google::protobuf::internal::Release_Store(&hint_, reinterpret_cast<google::protobuf::internal::AtomicWord>(block));
+    google::protobuf::internal::Release_Store(&hint_, reinterpret_cast<google::protobuf::internal::AtomicWord>(serial));
   }
 
-  google::protobuf::internal::AtomicWord threads_;          // Pointer to a linked list of ThreadInfo.
+  google::protobuf::internal::AtomicWord threads_;          // Pointer to a linked list of SerialArena.
   google::protobuf::internal::AtomicWord hint_;             // Fast thread-local block access
   google::protobuf::internal::AtomicWord space_allocated_;  // Sum of sizes of all allocated blocks.
 
   Block *initial_block_;     // If non-NULL, points to the block that came from
                              // user data.
 
-  // Returns a block owned by this thread.
-  Block* GetBlock(size_t n);
-  Block* GetBlockSlow(void* me, Block* my_full_block, size_t n);
-  Block* NewBlock(void* me, Block* my_last_block, size_t min_bytes);
-  void InitBlock(Block* b, void *me, size_t size);
-  static void* AllocFromBlock(Block* b, size_t n);
-  ThreadInfo* NewThreadInfo(Block* b);
-  ThreadInfo* FindThreadInfo(void* me);
-  ThreadInfo* GetThreadInfo(void* me, size_t n);
+  Block* NewBlock(Block* last_block, size_t min_bytes);
 
+  SerialArena* GetSerialArena();
+  bool GetSerialArenaFast(SerialArena** arena);
+  SerialArena* GetSerialArenaFallback(void* me);
   int64 lifecycle_id_;  // Unique for each arena. Changes on Reset().
 
   Options options_;
@@ -227,11 +299,15 @@ class LIBPROTOBUF_EXPORT ArenaImpl {
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(ArenaImpl);
 
  public:
-  // kHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8 to
-  // protect the invariant that pos is always at a multiple of 8.
-  static const size_t kHeaderSize = (sizeof(Block) + 7) & -8;
+  // kBlockHeaderSize is sizeof(Block), aligned up to the nearest multiple of 8
+  // to protect the invariant that pos is always at a multiple of 8.
+  static const size_t kBlockHeaderSize = (sizeof(Block) + 7) & -8;
+  static const size_t kSerialArenaSize = (sizeof(SerialArena) + 7) & -8;
 #if LANG_CXX11
-  static_assert(kHeaderSize % 8 == 0, "kHeaderSize must be a multiple of 8.");
+  static_assert(kBlockHeaderSize % 8 == 0,
+                "kBlockHeaderSize must be a multiple of 8.");
+  static_assert(kSerialArenaSize % 8 == 0,
+                "kSerialArenaSize must be a multiple of 8.");
 #endif
 };
 

src/google/protobuf/arena_unittest.cc

@@ -67,7 +67,6 @@ using protobuf_unittest::TestOneof2;
 using protobuf_unittest::TestEmptyMessage;
 
 namespace protobuf {
-namespace {
 
 class Notifier {
  public:
@@ -270,7 +269,7 @@ TEST(ArenaTest, InitialBlockTooSmall) {
   // Construct a small (64 byte) initial block of memory to be used by the
   // arena allocator; then, allocate an object which will not fit in the
   // initial block.
-  std::vector<char> arena_block(72);
+  std::vector<char> arena_block(96);
   ArenaOptions options;
   options.initial_block = &arena_block[0];
   options.initial_block_size = arena_block.size();
@@ -1299,12 +1298,12 @@ TEST(ArenaTest, SpaceAllocated_and_Used) {
   options.initial_block_size = 0;
   Arena arena_3(options);
   EXPECT_EQ(0, arena_3.SpaceUsed());
-  ::google::protobuf::Arena::CreateArray<char>(&arena_3, 182);
+  ::google::protobuf::Arena::CreateArray<char>(&arena_3, 160);
   EXPECT_EQ(256, arena_3.SpaceAllocated());
-  EXPECT_EQ(Align8(182), arena_3.SpaceUsed());
+  EXPECT_EQ(Align8(160), arena_3.SpaceUsed());
   ::google::protobuf::Arena::CreateArray<char>(&arena_3, 70);
   EXPECT_EQ(256 + 512, arena_3.SpaceAllocated());
-  EXPECT_EQ(Align8(182) + Align8(70), arena_3.SpaceUsed());
+  EXPECT_EQ(Align8(160) + Align8(70), arena_3.SpaceUsed());
   EXPECT_EQ(256 + 512, arena_3.Reset());
 }
 
@@ -1347,6 +1346,13 @@ TEST(ArenaTest, GetArenaShouldReturnNullForNonArenaAllocatedMessages) {
   EXPECT_EQ(NULL, Arena::GetArena(const_pointer_to_message));
 }
 
+TEST(ArenaTest, AddCleanup) {
+  ::google::protobuf::Arena arena;
+  for (int i = 0; i < 100; i++) {
+    arena.Own(new int);
+  }
+}
+
 TEST(ArenaTest, UnsafeSetAllocatedOnArena) {
   ::google::protobuf::Arena arena;
   TestAllTypes* message = Arena::CreateMessage<TestAllTypes>(&arena);
@@ -1405,13 +1411,20 @@ uint32 ArenaHooksTestUtil::num_reset = 0;
 uint32 ArenaHooksTestUtil::num_destruct = 0;
 const int ArenaHooksTestUtil::kCookieValue;
 
+class ArenaOptionsTestFriend {
+ public:
+  static void Set(::google::protobuf::ArenaOptions* options) {
+    options->on_arena_init = ArenaHooksTestUtil::on_init;
+    options->on_arena_allocation = ArenaHooksTestUtil::on_allocation;
+    options->on_arena_reset = ArenaHooksTestUtil::on_reset;
+    options->on_arena_destruction = ArenaHooksTestUtil::on_destruction;
+  }
+};
+
 // Test the hooks are correctly called and that the cookie is passed.
 TEST(ArenaTest, ArenaHooksSanity) {
   ::google::protobuf::ArenaOptions options;
-  options.on_arena_init = ArenaHooksTestUtil::on_init;
-  options.on_arena_allocation = ArenaHooksTestUtil::on_allocation;
-  options.on_arena_reset = ArenaHooksTestUtil::on_reset;
-  options.on_arena_destruction = ArenaHooksTestUtil::on_destruction;
+  ArenaOptionsTestFriend::Set(&options);
 
   // Scope for defining the arena
   {
@@ -1433,6 +1446,5 @@ TEST(ArenaTest, ArenaHooksSanity) {
 }
 
 
-}  // namespace
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/arenastring.h

@@ -51,6 +51,18 @@ namespace google {
 namespace protobuf {
 namespace internal {
 
+template <typename T>
+class TaggedPtr {
+ public:
+  void Set(T* p) { ptr_ = reinterpret_cast<uintptr_t>(p); }
+  T* Get() const { return reinterpret_cast<T*>(ptr_); }
+
+  bool IsNull() { return ptr_ == 0; }
+
+ private:
+  uintptr_t ptr_;
+};
+
 struct LIBPROTOBUF_EXPORT ArenaStringPtr {
   inline void Set(const ::std::string* default_value,
                   const ::std::string& value, ::google::protobuf::Arena* arena) {
@@ -294,6 +306,15 @@ struct LIBPROTOBUF_EXPORT ArenaStringPtr {
     return ptr_ == default_value;
   }
 
+  // Internal accessors!!!!
+  void UnsafeSetTaggedPointer(TaggedPtr< ::std::string> value) {
+    ptr_ = value.Get();
+  }
+  // Generated code only! An optimization, in certain cases the generated
+  // code is certain we can obtain a string with no default checks and
+  // tag tests.
+  ::std::string* UnsafeMutablePointer() { return ptr_; }
+
  private:
   ::std::string* ptr_;
 

src/google/protobuf/compiler/annotation_test_util.cc

@@ -76,10 +76,9 @@ void AddFile(const string& filename, const string& data) {
                              true));
 }
 
-bool CaptureMetadata(const string& filename, const string& plugin_specific_args,
-                     const string& meta_file_suffix, CommandLineInterface* cli,
-                     FileDescriptorProto* file,
-                     std::vector<ExpectedOutput>* outputs) {
+bool RunProtoCompiler(const string& filename,
+                      const string& plugin_specific_args,
+                      CommandLineInterface* cli, FileDescriptorProto* file) {
   cli->SetInputsAreProtoPathRelative(true);
 
   DescriptorCapturingGenerator capturing_generator(file);
@@ -92,24 +91,7 @@ bool CaptureMetadata(const string& filename, const string& plugin_specific_args,
                         plugin_specific_args.c_str(), capture_out.c_str(),
                         filename.c_str()};
 
-  if (cli->Run(5, argv) != 0) {
-    return false;
-  }
-
-  if (outputs != NULL) {
-    for (std::vector<ExpectedOutput>::iterator i = outputs->begin();
-         i != outputs->end(); ++i) {
-      GOOGLE_CHECK_OK(File::GetContents(TestTempDir() + "/" + i->file_path,
-                                 &i->file_content, true));
-      if (!DecodeMetadata(
-              TestTempDir() + "/" + i->file_path + meta_file_suffix,
-              &i->file_info)) {
-        return false;
-      }
-    }
-  }
-
-  return true;
+  return cli->Run(5, argv) == 0;
 }
 
 bool DecodeMetadata(const string& path, GeneratedCodeInfo* info) {

src/google/protobuf/compiler/annotation_test_util.h

@@ -59,25 +59,20 @@ struct ExpectedOutput {
 // directory.
 void AddFile(const string& filename, const string& data);
 
-// Tries to capture a FileDescriptorProto, GeneratedCodeInfo, and output
-// code from the previously added file with name `filename`.
+// Runs proto compiler. Captures proto file structrue in FileDescriptorProto.
+// Files will be generated in TestTempDir() folder. Callers of this
+// function must read generated files themselves.
 //
 // filename: source .proto file used to generate code.
 // plugin_specific_args: command line arguments specific to current generator.
 //     For Java, this value might be "--java_out=annotate_code:test_temp_dir"
-// meta_file_suffix: suffix of meta files that contain annotations. For Java
-//     it is ".pb.meta" because for file Foo.java meta file is Foo.java.pb.meta
 // cli: instance of command line interface to run generator. See Java's
 //     annotation_unittest.cc for an example of how to initialize it.
 // file: output parameter, will be set to the descriptor of the proto file
 //     specified in filename.
-// outputs: output parameter. If not NULL, each ExpectedOutput in the vector
-//     should have its file_path set; CaptureMetadata will fill the rest of
-//     the fields appropriately.
-bool CaptureMetadata(const string& filename, const string& plugin_specific_args,
-                     const string& meta_file_suffix, CommandLineInterface* cli,
-                     FileDescriptorProto* file,
-                     std::vector<ExpectedOutput>* outputs);
+bool RunProtoCompiler(const string& filename,
+                      const string& plugin_specific_args,
+                      CommandLineInterface* cli, FileDescriptorProto* file);
 
 bool DecodeMetadata(const string& path, GeneratedCodeInfo* info);
 

src/google/protobuf/compiler/cpp/cpp_enum_field.cc

@@ -195,7 +195,7 @@ GenerateInlineAccessorDefinitions(io::Printer* printer) const {
     "inline $type$ $classname$::$name$() const {\n"
     "  // @@protoc_insertion_point(field_get:$full_name$)\n"
     "  if (has_$name$()) {\n"
-    "    return static_cast< $type$ >($oneof_prefix$$name$_);\n"
+    "    return static_cast< $type$ >($field_member$);\n"
     "  }\n"
     "  return static_cast< $type$ >($default$);\n"
     "}\n"
@@ -209,14 +209,14 @@ GenerateInlineAccessorDefinitions(io::Printer* printer) const {
     "    clear_$oneof_name$();\n"
     "    set_has_$name$();\n"
     "  }\n"
-    "  $oneof_prefix$$name$_ = value;\n"
+    "  $field_member$ = value;\n"
     "  // @@protoc_insertion_point(field_set:$full_name$)\n"
     "}\n");
 }
 
 void EnumOneofFieldGenerator::
 GenerateClearingCode(io::Printer* printer) const {
-  printer->Print(variables_, "$oneof_prefix$$name$_ = $default$;\n");
+  printer->Print(variables_, "$field_member$ = $default$;\n");
 }
 
 void EnumOneofFieldGenerator::

src/google/protobuf/compiler/cpp/cpp_field.cc

@@ -67,12 +67,7 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
   (*variables)["number"] = SimpleItoa(descriptor->number());
   (*variables)["classname"] = ClassName(FieldScope(descriptor), false);
   (*variables)["declared_type"] = DeclaredTypeMethodName(descriptor->type());
-
-  // non_null_ptr_to_name is usable only if has_$name$ is true.  It yields a
-  // pointer that will not be NULL.  Subclasses of FieldGenerator may set
-  // (*variables)["non_null_ptr_to_name"] differently.
-  (*variables)["non_null_ptr_to_name"] =
-      StrCat("&this->", FieldName(descriptor), "()");
+  (*variables)["field_member"] = FieldName(descriptor) + "_";
 
   (*variables)["tag_size"] = SimpleItoa(
     WireFormat::TagSize(descriptor->number(), descriptor->type()));
@@ -81,8 +76,6 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
   (*variables)["deprecated_attr"] = descriptor->options().deprecated()
       ? "GOOGLE_PROTOBUF_DEPRECATED_ATTR " : "";
 
-  (*variables)["cppget"] = "Get";
-
   if (HasFieldPresence(descriptor->file())) {
     (*variables)["set_hasbit"] =
         "set_has_" + FieldName(descriptor) + "();";
@@ -93,10 +86,6 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
     (*variables)["clear_hasbit"] = "";
   }
 
-  // By default, empty string, so that generic code used for both oneofs and
-  // singular fields can be written.
-  (*variables)["oneof_prefix"] = "";
-
   // These variables are placeholders to pick out the beginning and ends of
   // identifiers for annotations (when doing so with existing variables would
   // be ambiguous or impossible). They should never be set to anything but the
@@ -108,10 +97,8 @@ void SetCommonFieldVariables(const FieldDescriptor* descriptor,
 void SetCommonOneofFieldVariables(const FieldDescriptor* descriptor,
                                   std::map<string, string>* variables) {
   const string prefix = descriptor->containing_oneof()->name() + "_.";
-  (*variables)["oneof_prefix"] = prefix;
   (*variables)["oneof_name"] = descriptor->containing_oneof()->name();
-  (*variables)["non_null_ptr_to_name"] =
-      StrCat(prefix, (*variables)["name"], "_");
+  (*variables)["field_member"] = StrCat(prefix, (*variables)["name"], "_");
 }
 
 FieldGenerator::~FieldGenerator() {}

src/google/protobuf/compiler/cpp/cpp_field.h

@@ -184,6 +184,10 @@ class FieldGenerator {
   // message's MergeFromCodedStream() method.
   virtual void GenerateMergeFromCodedStream(io::Printer* printer) const = 0;
 
+  // Returns true if this field's "MergeFromCodedStream" code needs the arena
+  // to be defined as a variable.
+  virtual bool MergeFromCodedStreamNeedsArena() const { return false; }
+
   // Generate lines to decode this field from a packed value, which will be
   // placed inside the message's MergeFromCodedStream() method.
   virtual void GenerateMergeFromCodedStreamWithPacking(io::Printer* printer)

src/google/protobuf/compiler/cpp/cpp_file.cc

@@ -397,7 +397,7 @@ void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
 
     // Define default instances
     GenerateSourceDefaultInstance(idx, printer);
-    if (UsingImplicitWeakFields(file_, options_)) {
+    if (options_.lite_implicit_weak_fields) {
       printer->Print("void $classname$_ReferenceStrong() {}\n", "classname",
                      message_generators_[idx]->classname_);
     }
@@ -416,6 +416,13 @@ void FileGenerator::GenerateSourceForMessage(int idx, io::Printer* printer) {
     GenerateInitForSCC(GetSCC(message_generators_[idx]->descriptor_), printer);
   }
 
+
+  printer->Print(
+      "namespace google {\nnamespace protobuf {\n");
+  message_generators_[idx]->GenerateSourceInProto2Namespace(printer);
+  printer->Print(
+      "}  // namespace protobuf\n}  // namespace google\n");
+
   printer->Print(
       "\n"
       "// @@protoc_insertion_point(global_scope)\n");
@@ -467,7 +474,7 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
     // Define default instances
     for (int i = 0; i < message_generators_.size(); i++) {
       GenerateSourceDefaultInstance(i, printer);
-      if (UsingImplicitWeakFields(file_, options_)) {
+      if (options_.lite_implicit_weak_fields) {
         printer->Print("void $classname$_ReferenceStrong() {}\n", "classname",
                        message_generators_[i]->classname_);
       }
@@ -525,6 +532,15 @@ void FileGenerator::GenerateSource(io::Printer* printer) {
       "\n"
       "// @@protoc_insertion_point(namespace_scope)\n");
   }
+
+  printer->Print(
+      "namespace google {\nnamespace protobuf {\n");
+  for (int i = 0; i < message_generators_.size(); i++) {
+    message_generators_[i]->GenerateSourceInProto2Namespace(printer);
+  }
+  printer->Print(
+      "}  // namespace protobuf\n}  // namespace google\n");
+
   printer->Print(
     "\n"
     "// @@protoc_insertion_point(global_scope)\n");
@@ -553,7 +569,42 @@ class FileGenerator::ForwardDeclarations {
   std::map<string, const Descriptor*>& classes() { return classes_; }
   std::map<string, const EnumDescriptor*>& enums() { return enums_; }
 
-  void Print(io::Printer* printer, const Options& options) const {
+  void PrintForwardDeclarations(io::Printer* printer,
+                                const Options& options) const {
+    PrintNestedDeclarations(printer, options);
+    PrintTopLevelDeclarations(printer, options);
+  }
+
+
+ private:
+  void PrintNestedDeclarations(io::Printer* printer,
+                               const Options& options) const {
+    PrintDeclarationsInsideNamespace(printer, options);
+    for (std::map<string, ForwardDeclarations *>::const_iterator
+             it = namespaces_.begin(),
+             end = namespaces_.end();
+         it != end; ++it) {
+      printer->Print("namespace $nsname$ {\n",
+                     "nsname", it->first);
+      it->second->PrintNestedDeclarations(printer, options);
+      printer->Print("}  // namespace $nsname$\n",
+                     "nsname", it->first);
+    }
+  }
+
+  void PrintTopLevelDeclarations(io::Printer* printer,
+                                 const Options& options) const {
+    PrintDeclarationsOutsideNamespace(printer, options);
+    for (std::map<string, ForwardDeclarations *>::const_iterator
+             it = namespaces_.begin(),
+             end = namespaces_.end();
+         it != end; ++it) {
+      it->second->PrintTopLevelDeclarations(printer, options);
+    }
+  }
+
+  void PrintDeclarationsInsideNamespace(io::Printer* printer,
+                                        const Options& options) const {
     for (std::map<string, const EnumDescriptor *>::const_iterator
              it = enums_.begin(),
              end = enums_.end();
@@ -584,20 +635,36 @@ class FileGenerator::ForwardDeclarations {
                        "classname", it->first);
       }
     }
-    for (std::map<string, ForwardDeclarations *>::const_iterator
-             it = namespaces_.begin(),
-             end = namespaces_.end();
+  }
+
+  void PrintDeclarationsOutsideNamespace(io::Printer* printer,
+                                         const Options& options) const {
+    if (classes_.size() == 0) return;
+
+    printer->Print(
+        "namespace google {\nnamespace protobuf {\n");
+    for (std::map<string, const Descriptor*>::const_iterator
+             it = classes_.begin(),
+             end = classes_.end();
          it != end; ++it) {
-      printer->Print("namespace $nsname$ {\n",
-                     "nsname", it->first);
-      it->second->Print(printer, options);
-      printer->Print("}  // namespace $nsname$\n",
-                     "nsname", it->first);
+      const Descriptor* d = it->second;
+      string extra_class_qualifier;
+      // "class" is to disambiguate in case there is also a function with this
+      // name.  There is code out there that does this!
+      printer->Print(
+          "template<> "
+          "$dllexport_decl$"
+          "$class$$classname$* Arena::$func$< $class$$classname$>(Arena*);\n",
+          "classname", QualifiedClassName(d),
+          "func", MessageCreateFunction(d),
+          "class", extra_class_qualifier,
+          "dllexport_decl",
+          options.dllexport_decl.empty() ? "" : options.dllexport_decl + " ");
     }
+    printer->Print(
+        "}  // namespace protobuf\n}  // namespace google\n");
   }
 
-
- private:
   std::map<string, ForwardDeclarations*> namespaces_;
   std::map<string, const Descriptor*> classes_;
   std::map<string, const EnumDescriptor*> enums_;
@@ -1053,7 +1120,7 @@ void FileGenerator::GenerateInitializationCode(io::Printer* printer) {
 void FileGenerator::GenerateForwardDeclarations(io::Printer* printer) {
   ForwardDeclarations decls;
   FillForwardDeclarations(&decls);
-  decls.Print(printer, options_);
+  decls.PrintForwardDeclarations(printer, options_);
 }
 
 void FileGenerator::FillForwardDeclarations(ForwardDeclarations* decls) {

src/google/protobuf/compiler/cpp/cpp_generator.cc

@@ -41,12 +41,12 @@
 #endif
 #include <utility>
 
+#include <google/protobuf/stubs/strutil.h>
 #include <google/protobuf/compiler/cpp/cpp_file.h>
 #include <google/protobuf/compiler/cpp/cpp_helpers.h>
 #include <google/protobuf/io/printer.h>
 #include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/descriptor.pb.h>
-#include <google/protobuf/stubs/strutil.h>
 
 namespace google {
 namespace protobuf {
@@ -85,7 +85,6 @@ bool CppGenerator::Generate(const FileDescriptor* file,
   // __declspec(dllimport) depending on what is being compiled.
   //
   Options file_options;
-  bool split_source = false;
   for (int i = 0; i < options.size(); i++) {
     if (options[i].first == "dllexport_decl") {
       file_options.dllexport_decl = options[i].second;
@@ -101,18 +100,28 @@ bool CppGenerator::Generate(const FileDescriptor* file,
       file_options.enforce_lite = true;
     } else if (options[i].first == "lite_implicit_weak_fields") {
       file_options.lite_implicit_weak_fields = true;
+      if (!options[i].second.empty()) {
+        file_options.num_cc_files = strto32(options[i].second.c_str(),
+                                            NULL, 10);
+      }
     } else if (options[i].first == "table_driven_parsing") {
       file_options.table_driven_parsing = true;
     } else if (options[i].first == "table_driven_serialization") {
       file_options.table_driven_serialization = true;
-    } else if (options[i].first == "split_source") {
-      split_source = true;
     } else {
       *error = "Unknown generator option: " + options[i].first;
       return false;
     }
   }
 
+  // The safe_boundary_check option controls behavior for Google-internal
+  // protobuf APIs.
+  if (file_options.safe_boundary_check) {
+    *error =
+        "The safe_boundary_check option is not supported outside of Google.";
+    return false;
+  }
+
   // -----------------------------------------------------------------
 
 
@@ -159,8 +168,8 @@ bool CppGenerator::Generate(const FileDescriptor* file,
     }
   }
 
-  // Generate cc file.
-  if (split_source) {
+  // Generate cc file(s).
+  if (UsingImplicitWeakFields(file, file_options)) {
     {
       // This is the global .cc file, containing enum/services/tables/reflection
       google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
@@ -168,12 +177,26 @@ bool CppGenerator::Generate(const FileDescriptor* file,
       io::Printer printer(output.get(), '$');
       file_generator.GenerateGlobalSource(&printer);
     }
-    for (int i = 0; i < file_generator.NumMessages(); i++) {
+
+    int num_cc_files = file_generator.NumMessages();
+
+    // If we're using implicit weak fields then we allow the user to optionally
+    // specify how many files to generate, not counting the global pb.cc file.
+    // If we have more files than messages, then some files will be generated as
+    // empty placeholders.
+    if (file_options.num_cc_files > 0) {
+      GOOGLE_CHECK_LE(file_generator.NumMessages(), file_options.num_cc_files)
+          << "There must be at least as many numbered .cc files as messages.";
+      num_cc_files = file_options.num_cc_files;
+    }
+    for (int i = 0; i < num_cc_files; i++) {
       // TODO(gerbens) Agree on naming scheme.
       google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(
           generator_context->Open(basename + "." + SimpleItoa(i) + ".cc"));
       io::Printer printer(output.get(), '$');
-      file_generator.GenerateSourceForMessage(i, &printer);
+      if (i < file_generator.NumMessages()) {
+        file_generator.GenerateSourceForMessage(i, &printer);
+      }
     }
   } else {
     google::protobuf::scoped_ptr<io::ZeroCopyOutputStream> output(

src/google/protobuf/compiler/cpp/cpp_helpers.cc

@@ -752,7 +752,7 @@ bool UsingImplicitWeakFields(const FileDescriptor* file,
 bool IsImplicitWeakField(const FieldDescriptor* field, const Options& options) {
   return UsingImplicitWeakFields(field->file(), options) &&
          field->type() == FieldDescriptor::TYPE_MESSAGE &&
-         !field->is_required() && !field->is_repeated() && !field->is_map() &&
+         !field->is_required() && !field->is_map() &&
          field->containing_oneof() == NULL;
 }
 

src/google/protobuf/compiler/cpp/cpp_helpers.h

@@ -310,6 +310,10 @@ inline bool IsCrossFileMessage(const FieldDescriptor* field) {
          field->message_type()->file() != field->file();
 }
 
+inline string MessageCreateFunction(const Descriptor* d) {
+  return SupportsArenas(d) ? "CreateMessage" : "Create";
+}
+
 bool IsAnyMessage(const FileDescriptor* descriptor);
 bool IsAnyMessage(const Descriptor* descriptor);
 

src/google/protobuf/compiler/cpp/cpp_message.h

@@ -106,6 +106,9 @@ class MessageGenerator {
   // Generate all non-inline methods for this class.
   void GenerateClassMethods(io::Printer* printer);
 
+  // Generate source file code that should go outside any namespace.
+  void GenerateSourceInProto2Namespace(io::Printer* printer);
+
  private:
   // Generate declarations and definitions of accessors for fields.
   void GenerateDependentBaseClassDefinition(io::Printer* printer);

src/google/protobuf/compiler/cpp/cpp_message_field.h

@@ -54,9 +54,7 @@ class MessageFieldGenerator : public FieldGenerator {
 
   // implements FieldGenerator ---------------------------------------
   void GeneratePrivateMembers(io::Printer* printer) const;
-  void GenerateDependentAccessorDeclarations(io::Printer* printer) const;
   void GenerateAccessorDeclarations(io::Printer* printer) const;
-  void GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
@@ -73,7 +71,6 @@ class MessageFieldGenerator : public FieldGenerator {
 
  protected:
   const FieldDescriptor* descriptor_;
-  const bool dependent_field_;
   const bool implicit_weak_field_;
   std::map<string, string> variables_;
 
@@ -88,7 +85,6 @@ class MessageOneofFieldGenerator : public MessageFieldGenerator {
   ~MessageOneofFieldGenerator();
 
   // implements FieldGenerator ---------------------------------------
-  void GenerateDependentInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateNonInlineAccessorDefinitions(io::Printer* printer) const;
   void GenerateClearingCode(io::Printer* printer) const;
@@ -101,9 +97,6 @@ class MessageOneofFieldGenerator : public MessageFieldGenerator {
   void GenerateConstructorCode(io::Printer* printer) const;
 
  private:
-  void InternalGenerateInlineAccessorDefinitions(
-      const std::map<string, string>& variables, io::Printer* printer) const;
-
   const bool dependent_base_;
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(MessageOneofFieldGenerator);
 };
@@ -136,7 +129,7 @@ class RepeatedMessageFieldGenerator : public FieldGenerator {
 
   const FieldDescriptor* descriptor_;
   const bool dependent_field_;
-  const bool dependent_getter_;
+  const bool implicit_weak_field_;
   std::map<string, string> variables_;
 
   GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(RepeatedMessageFieldGenerator);

src/google/protobuf/compiler/cpp/cpp_options.h

@@ -54,6 +54,7 @@ struct Options {
         table_driven_parsing(false),
         table_driven_serialization(false),
         lite_implicit_weak_fields(false),
+        num_cc_files(0),
         access_info_map(NULL) {}
 
   string dllexport_decl;
@@ -65,6 +66,7 @@ struct Options {
   bool table_driven_parsing;
   bool table_driven_serialization;
   bool lite_implicit_weak_fields;
+  int num_cc_files;
   string annotation_pragma_name;
   string annotation_guard_name;
   const AccessInfoMap* access_info_map;

src/google/protobuf/compiler/cpp/cpp_string_field.h

@@ -69,6 +69,8 @@ class StringFieldGenerator : public FieldGenerator {
   void GenerateSerializeWithCachedSizesToArray(io::Printer* printer) const;
   void GenerateByteSize(io::Printer* printer) const;
 
+  bool MergeFromCodedStreamNeedsArena() const;
+
  protected:
   const FieldDescriptor* descriptor_;
   std::map<string, string> variables_;