Merge pull request #6161 from haon4/201905211601

Down integrate to GitHub

benchmarks/util/result_parser.py

@@ -18,7 +18,7 @@ def __get_data_size(filename):
     return __file_size_map[filename]
   benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
   benchmark_dataset.ParseFromString(
-      open(filename).read())
+      open(filename, "rb").read())
   size = 0
   count = 0
   for payload in benchmark_dataset.payload:
@@ -60,7 +60,7 @@ def __parse_cpp_result(filename):
     return
   if filename[0] != '/':
     filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     results = json.loads(f.read())
     for benchmark in results["benchmarks"]:
       data_filename = "".join(
@@ -95,7 +95,7 @@ def __parse_synthetic_result(filename):
     return
   if filename[0] != "/":
     filename = os.path.dirname(os.path.abspath(__file__)) + "/" + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     results = json.loads(f.read())
     for benchmark in results["benchmarks"]:
       __results.append({
@@ -125,7 +125,7 @@ def __parse_python_result(filename):
     return
   if filename[0] != '/':
     filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     results_list = json.loads(f.read())
     for results in results_list:
       for result in results:
@@ -175,7 +175,7 @@ def __parse_java_result(filename):
     return
   if filename[0] != '/':
     filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     results = json.loads(f.read())
     for result in results:
       total_weight = 0
@@ -211,7 +211,7 @@ def __parse_go_result(filename):
     return
   if filename[0] != '/':
     filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     for line in f:
       result_list = re.split(r"[\ \t]+", line)
       if result_list[0][:9] != "Benchmark":
@@ -251,7 +251,7 @@ def __parse_custom_result(filename, language):
     return
   if filename[0] != '/':
     filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
-  with open(filename) as f:
+  with open(filename, "rb") as f:
     results = json.loads(f.read())
     for result in results:
       _, avg_size = __get_data_size(result["filename"])

csharp/src/Google.Protobuf/Reflection/Descriptor.cs

@@ -1194,6 +1194,9 @@ namespace Google.Protobuf.Reflection {
         private readonly static int StartDefaultValue = 0;
 
         private int start_;
+        /// <summary>
+        /// Inclusive.
+        /// </summary>
         [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
         public int Start {
           get { if ((_hasBits0 & 1) != 0) { return start_; } else { return StartDefaultValue; } }
@@ -1218,6 +1221,9 @@ namespace Google.Protobuf.Reflection {
         private readonly static int EndDefaultValue = 0;
 
         private int end_;
+        /// <summary>
+        /// Exclusive.
+        /// </summary>
         [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
         public int End {
           get { if ((_hasBits0 & 2) != 0) { return end_; } else { return EndDefaultValue; } }

java/core/src/main/java/com/google/protobuf/AbstractMessageLite.java

@@ -51,6 +51,7 @@ public abstract class AbstractMessageLite<
         BuilderType extends AbstractMessageLite.Builder<MessageType, BuilderType>>
     implements MessageLite {
   protected int memoizedHashCode = 0;
+
   @Override
   public ByteString toByteString() {
     try {
@@ -106,16 +107,6 @@ public abstract class AbstractMessageLite<
   }
 
 
-  @ExperimentalApi
-  protected final boolean isInitializedInternal() {
-    return Protobuf.getInstance().schemaFor(this).isInitialized(this);
-  }
-
-  @ExperimentalApi
-  protected final int getSerializedSizeInternal() {
-    return Protobuf.getInstance().schemaFor(this).getSerializedSize(this);
-  }
-
   int getSerializedSize(Schema schema) {
     int memoizedSerializedSize = getMemoizedSerializedSize();
     if (memoizedSerializedSize == -1) {
@@ -125,37 +116,6 @@ public abstract class AbstractMessageLite<
     return memoizedSerializedSize;
   }
 
-  @ExperimentalApi
-  protected final void writeToInternal(CodedOutputStream output) throws IOException {
-    Protobuf.getInstance()
-        .schemaFor(getClassInternal())
-        .writeTo(this, CodedOutputStreamWriter.forCodedOutput(output));
-  }
-
-  @ExperimentalApi
-  protected void mergeFromInternal(CodedInputStream input, ExtensionRegistryLite extensionRegistry)
-      throws InvalidProtocolBufferException {
-    try {
-      Protobuf.getInstance()
-          .schemaFor(getClassInternal())
-          .mergeFrom(this, CodedInputStreamReader.forCodedInput(input), extensionRegistry);
-    } catch (InvalidProtocolBufferException e) {
-      throw e.setUnfinishedMessage(this);
-    } catch (IOException e) {
-      throw new InvalidProtocolBufferException(e).setUnfinishedMessage(this);
-    }
-  }
-
-  @ExperimentalApi
-  protected void makeImmutableInternal() {
-    Protobuf.getInstance().schemaFor(getClassInternal()).makeImmutable(this);
-  }
-
-  @SuppressWarnings("unchecked")
-  private Class<AbstractMessageLite<MessageType, BuilderType>> getClassInternal() {
-    return (Class<AbstractMessageLite<MessageType, BuilderType>>) getClass();
-  }
-
   /** Package private helper method for AbstractParser to create UninitializedMessageException. */
   UninitializedMessageException newUninitializedMessageException() {
     return new UninitializedMessageException(this);

java/core/src/main/java/com/google/protobuf/GeneratedMessageLite.java

@@ -262,12 +262,14 @@ public abstract class GeneratedMessageLite<
   }
 
   public void writeTo(CodedOutputStream output) throws IOException {
-    writeToInternal(output);
+    Protobuf.getInstance()
+        .schemaFor(this)
+        .writeTo(this, CodedOutputStreamWriter.forCodedOutput(output));
   }
 
   public int getSerializedSize() {
     if (memoizedSerializedSize == -1) {
-      memoizedSerializedSize = getSerializedSizeInternal();
+      memoizedSerializedSize = Protobuf.getInstance().schemaFor(this).getSerializedSize(this);
     }
     return memoizedSerializedSize;
   }

java/core/src/main/java/com/google/protobuf/GeneratedMessageV3.java

@@ -126,6 +126,21 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
     return internalGetFieldAccessorTable().descriptor;
   }
 
+  protected void mergeFromAndMakeImmutableInternal(
+      CodedInputStream input, ExtensionRegistryLite extensionRegistry)
+      throws InvalidProtocolBufferException {
+    Schema<GeneratedMessageV3> schema =
+        (Schema<GeneratedMessageV3>) Protobuf.getInstance().schemaFor(this);
+    try {
+      schema.mergeFrom(this, CodedInputStreamReader.forCodedInput(input), extensionRegistry);
+    } catch (InvalidProtocolBufferException e) {
+      throw e.setUnfinishedMessage(this);
+    } catch (IOException e) {
+      throw new InvalidProtocolBufferException(e).setUnfinishedMessage(this);
+    }
+    schema.makeImmutable(this);
+  }
+
   /**
    * Internal helper to return a modifiable map containing all the fields.
    * The returned Map is modifialbe so that the caller can add additional
@@ -439,6 +454,7 @@ public abstract class GeneratedMessageV3 extends AbstractMessage
         size == 0 ? AbstractProtobufList.DEFAULT_CAPACITY : size * 2);
   }
 
+
   @Override
   public void writeTo(final CodedOutputStream output) throws IOException {
     MessageReflection.writeMessageTo(this, getAllFieldsRaw(), output, false);

java/lite/src/test/java/com/google/protobuf/LiteTest.java

@@ -99,7 +99,7 @@ public class LiteTest extends TestCase {
         TestAllTypesLite.newBuilder()
             .setOptionalInt32(123)
             .addRepeatedString("hello")
-            .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(7))
+            .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(7))
             .build();
 
     ByteString data = message.toByteString();
@@ -135,11 +135,10 @@ public class LiteTest extends TestCase {
         TestAllExtensionsLite.newBuilder()
             .setExtension(UnittestLite.optionalInt32ExtensionLite, 123)
             .addExtension(UnittestLite.repeatedStringExtensionLite, "hello")
-            .setExtension(
-                UnittestLite.optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ)
+            .setExtension(UnittestLite.optionalNestedEnumExtensionLite, NestedEnum.BAZ)
             .setExtension(
                 UnittestLite.optionalNestedMessageExtensionLite,
-                TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build())
+                NestedMessage.newBuilder().setBb(7).build())
             .build();
 
     // Test copying a message, since coping extensions actually does use a
@@ -152,8 +151,7 @@ public class LiteTest extends TestCase {
     assertEquals(1, message2.getExtension(UnittestLite.repeatedStringExtensionLite).size());
     assertEquals("hello", message2.getExtension(UnittestLite.repeatedStringExtensionLite, 0));
     assertEquals(
-        TestAllTypesLite.NestedEnum.BAZ,
-        message2.getExtension(UnittestLite.optionalNestedEnumExtensionLite));
+        NestedEnum.BAZ, message2.getExtension(UnittestLite.optionalNestedEnumExtensionLite));
     assertEquals(7, message2.getExtension(UnittestLite.optionalNestedMessageExtensionLite).getBb());
   }
 
@@ -1377,14 +1375,16 @@ public class LiteTest extends TestCase {
     proto =
         TestAllTypesLite.newBuilder()
             .setOptionalBool(true)
-            .setOptionalNestedEnum(TestAllTypesLite.NestedEnum.BAZ)
+            .setOptionalNestedEnum(NestedEnum.BAZ)
             .build();
-    assertToStringEquals("optional_bool: true\noptional_nested_enum: BAZ", proto);
+    assertToStringEquals(
+        "optional_bool: true\noptional_nested_enum: " + NestedEnum.BAZ.toString(), proto);
 
     proto = TestAllTypesLite.newBuilder().setOptionalFloat(2.72f).setOptionalDouble(3.14).build();
     assertToStringEquals("optional_double: 3.14\noptional_float: 2.72", proto);
   }
 
+
   public void testToStringStringFields() throws Exception {
     TestAllTypesLite proto =
         TestAllTypesLite.newBuilder().setOptionalString("foo\"bar\nbaz\\").build();
@@ -1397,13 +1397,13 @@ public class LiteTest extends TestCase {
   public void testToStringNestedMessage() throws Exception {
     TestAllTypesLite proto =
         TestAllTypesLite.newBuilder()
-            .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.getDefaultInstance())
+            .setOptionalNestedMessage(NestedMessage.getDefaultInstance())
             .build();
     assertToStringEquals("optional_nested_message {\n}", proto);
 
     proto =
         TestAllTypesLite.newBuilder()
-            .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(7))
+            .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(7))
             .build();
     assertToStringEquals("optional_nested_message {\n  bb: 7\n}", proto);
   }
@@ -1419,8 +1419,8 @@ public class LiteTest extends TestCase {
 
     proto =
         TestAllTypesLite.newBuilder()
-            .addRepeatedLazyMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(7))
-            .addRepeatedLazyMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(8))
+            .addRepeatedLazyMessage(NestedMessage.newBuilder().setBb(7))
+            .addRepeatedLazyMessage(NestedMessage.newBuilder().setBb(8))
             .build();
     assertToStringEquals(
         "repeated_lazy_message {\n  bb: 7\n}\nrepeated_lazy_message {\n  bb: 8\n}", proto);
@@ -1433,7 +1433,10 @@ public class LiteTest extends TestCase {
             .setOptionalForeignMessage(ForeignMessageLite.newBuilder().setC(3))
             .build();
     assertToStringEquals(
-        "optional_foreign_enum: FOREIGN_LITE_BAR\noptional_foreign_message {\n  c: 3\n}", proto);
+        "optional_foreign_enum: "
+            + ForeignEnumLite.FOREIGN_LITE_BAR
+            + "\noptional_foreign_message {\n  c: 3\n}",
+        proto);
   }
 
   public void testToStringExtensions() throws Exception {
@@ -1442,11 +1445,10 @@ public class LiteTest extends TestCase {
             .setExtension(UnittestLite.optionalInt32ExtensionLite, 123)
             .addExtension(UnittestLite.repeatedStringExtensionLite, "spam")
             .addExtension(UnittestLite.repeatedStringExtensionLite, "eggs")
-            .setExtension(
-                UnittestLite.optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ)
+            .setExtension(UnittestLite.optionalNestedEnumExtensionLite, NestedEnum.BAZ)
             .setExtension(
                 UnittestLite.optionalNestedMessageExtensionLite,
-                TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build())
+                NestedMessage.newBuilder().setBb(7).build())
             .build();
     assertToStringEquals(
         "[1]: 123\n[18] {\n  bb: 7\n}\n[21]: 3\n[44]: \"spam\"\n[44]: \"eggs\"", message);
@@ -1458,11 +1460,10 @@ public class LiteTest extends TestCase {
             .setExtension(UnittestLite.optionalInt32ExtensionLite, 123)
             .addExtension(UnittestLite.repeatedStringExtensionLite, "spam")
             .addExtension(UnittestLite.repeatedStringExtensionLite, "eggs")
-            .setExtension(
-                UnittestLite.optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ)
+            .setExtension(UnittestLite.optionalNestedEnumExtensionLite, NestedEnum.BAZ)
             .setExtension(
                 UnittestLite.optionalNestedMessageExtensionLite,
-                TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build())
+                NestedMessage.newBuilder().setBb(7).build())
             .build();
     TestAllExtensionsLite messageWithUnknownFields =
         TestAllExtensionsLite.parseFrom(messageWithExtensions.toByteArray());
@@ -1611,13 +1612,12 @@ public class LiteTest extends TestCase {
 
   public void testMergeFromNoLazyFieldSharing() throws Exception {
     TestAllTypesLite.Builder sourceBuilder =
-        TestAllTypesLite.newBuilder()
-            .setOptionalLazyMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(1));
+        TestAllTypesLite.newBuilder().setOptionalLazyMessage(NestedMessage.newBuilder().setBb(1));
     TestAllTypesLite.Builder targetBuilder =
         TestAllTypesLite.newBuilder().mergeFrom(sourceBuilder.build());
     assertEquals(1, sourceBuilder.getOptionalLazyMessage().getBb());
     // now change the sourceBuilder, and target value shouldn't be affected.
-    sourceBuilder.setOptionalLazyMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(2));
+    sourceBuilder.setOptionalLazyMessage(NestedMessage.newBuilder().setBb(2));
     assertEquals(1, targetBuilder.getOptionalLazyMessage().getBb());
   }
 
@@ -2115,7 +2115,7 @@ public class LiteTest extends TestCase {
         TestAllTypesLite.newBuilder()
             .setOptionalInt32(123)
             .addRepeatedString("hello")
-            .setOptionalNestedMessage(TestAllTypesLite.NestedMessage.newBuilder().setBb(7))
+            .setOptionalNestedMessage(NestedMessage.newBuilder().setBb(7))
             .build();
 
     TestAllTypesLite copy =
@@ -2150,11 +2150,10 @@ public class LiteTest extends TestCase {
         TestAllExtensionsLite.newBuilder()
             .setExtension(UnittestLite.optionalInt32ExtensionLite, 123)
             .addExtension(UnittestLite.repeatedStringExtensionLite, "hello")
-            .setExtension(
-                UnittestLite.optionalNestedEnumExtensionLite, TestAllTypesLite.NestedEnum.BAZ)
+            .setExtension(UnittestLite.optionalNestedEnumExtensionLite, NestedEnum.BAZ)
             .setExtension(
                 UnittestLite.optionalNestedMessageExtensionLite,
-                TestAllTypesLite.NestedMessage.newBuilder().setBb(7).build())
+                NestedMessage.newBuilder().setBb(7).build())
             .build();
 
     ExtensionRegistryLite registry = ExtensionRegistryLite.newInstance();

php/src/Google/Protobuf/Internal/DescriptorProto/ExtensionRange.php

@@ -16,11 +16,15 @@ use Google\Protobuf\Internal\GPBUtil;
 class ExtensionRange extends \Google\Protobuf\Internal\Message
 {
     /**
+     * Inclusive.
+     *
      * Generated from protobuf field <code>optional int32 start = 1;</code>
      */
     private $start = 0;
     private $has_start = false;
     /**
+     * Exclusive.
+     *
      * Generated from protobuf field <code>optional int32 end = 2;</code>
      */
     private $end = 0;
@@ -38,7 +42,9 @@ class ExtensionRange extends \Google\Protobuf\Internal\Message
      *     Optional. Data for populating the Message object.
      *
      *     @type int $start
+     *           Inclusive.
      *     @type int $end
+     *           Exclusive.
      *     @type \Google\Protobuf\Internal\ExtensionRangeOptions $options
      * }
      */
@@ -48,6 +54,8 @@ class ExtensionRange extends \Google\Protobuf\Internal\Message
     }
 
     /**
+     * Inclusive.
+     *
      * Generated from protobuf field <code>optional int32 start = 1;</code>
      * @return int
      */
@@ -57,6 +65,8 @@ class ExtensionRange extends \Google\Protobuf\Internal\Message
     }
 
     /**
+     * Inclusive.
+     *
      * Generated from protobuf field <code>optional int32 start = 1;</code>
      * @param int $var
      * @return $this
@@ -76,6 +86,8 @@ class ExtensionRange extends \Google\Protobuf\Internal\Message
     }
 
     /**
+     * Exclusive.
+     *
      * Generated from protobuf field <code>optional int32 end = 2;</code>
      * @return int
      */
@@ -85,6 +97,8 @@ class ExtensionRange extends \Google\Protobuf\Internal\Message
     }
 
     /**
+     * Exclusive.
+     *
      * Generated from protobuf field <code>optional int32 end = 2;</code>
      * @param int $var
      * @return $this

src/google/protobuf/compiler/js/js_generator.cc

@@ -3731,7 +3731,12 @@ bool Generator::GenerateAll(const std::vector<const FileDescriptor*>& files,
                            options.GetFileNameExtension();
     std::unique_ptr<io::ZeroCopyOutputStream> output(context->Open(filename));
     GOOGLE_CHECK(output.get());
-    io::Printer printer(output.get(), '$');
+    GeneratedCodeInfo annotations;
+    io::AnnotationProtoCollector<GeneratedCodeInfo> annotation_collector(
+        &annotations);
+    io::Printer printer(
+        output.get(), '$',
+        options.annotate_code ? &annotation_collector : nullptr);
 
     // Pull out all extensions -- we need these to generate all
     // provides/requires.
@@ -3763,6 +3768,9 @@ bool Generator::GenerateAll(const std::vector<const FileDescriptor*>& files,
     if (printer.failed()) {
       return false;
     }
+    if (options.annotate_code) {
+      EmbedCodeAnnotations(annotations, &printer);
+    }
   } else if (options.output_mode() == GeneratorOptions::kOneOutputFilePerSCC) {
     std::set<const Descriptor*> have_printed;
     SCCAnalyzer<DepsGenerator> analyzer;

src/google/protobuf/descriptor.proto

@@ -101,8 +101,8 @@ message DescriptorProto {
   repeated EnumDescriptorProto enum_type = 4;
 
   message ExtensionRange {
-    optional int32 start = 1;
-    optional int32 end = 2;
+    optional int32 start = 1;  // Inclusive.
+    optional int32 end = 2;    // Exclusive.
 
     optional ExtensionRangeOptions options = 3;
   }

src/google/protobuf/message_lite.h

@@ -246,6 +246,9 @@ class PROTOBUF_EXPORT MessageLite {
   // assume it will remain stable over time.
   std::string DebugString() const;
   std::string ShortDebugString() const { return DebugString(); }
+  // MessageLite::DebugString is already Utf8 Safe. This is to add compatibility
+  // with Message.
+  std::string Utf8DebugString() const { return DebugString(); }
 
   // Parsing ---------------------------------------------------------
   // Methods for parsing in protocol buffer format.  Most of these are