Down-integrate from internal code base.

src/google/protobuf/compiler/cpp/cpp_message.cc

@@ -561,7 +561,7 @@ GenerateFieldAccessorDefinitions(io::Printer* printer) {
           } else {
             printer->Print(vars,
               "inline bool $classname$::has_$name$() const {\n"
-              "  return $name$_ != NULL;\n"
+              "  return !_is_default_instance_ && $name$_ != NULL;\n"
               "}\n");
           }
         }
@@ -1051,6 +1051,22 @@ GenerateClassDefinition(io::Printer* printer) {
       printer->Print(cached_size_decl.c_str());
       need_to_emit_cached_size = false;
     }
+  } else {
+    // Without field presence, we need another way to disambiguate the default
+    // instance, because the default instance's submessage fields (if any) store
+    // pointers to the default instances of the submessages even when they
+    // aren't present. Alternatives to this approach might be to (i) use a
+    // tagged pointer on all message fields, setting a tag bit for "not really
+    // present, just default instance"; or (ii) comparing |this| against the
+    // return value from GeneratedMessageFactory::GetPrototype() in all
+    // has_$field$() calls. However, both of these options are much more
+    // expensive (in code size and CPU overhead) than just checking a field in
+    // the message. Long-term, the best solution would be to rearchitect the
+    // default instance design not to store pointers to submessage default
+    // instances, and have reflection get those some other way; but that change
+    // would have too much impact on proto2.
+    printer->Print(
+      "bool _is_default_instance_;\n");
   }
 
   // Field members:
@@ -1323,11 +1339,21 @@ GenerateDescriptorInitializer(io::Printer* printer, int index) {
   if (UseUnknownFieldSet(descriptor_->file())) {
     printer->Print(vars,
     "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET("
-    "$classname$, _internal_metadata_));\n");
+    "$classname$, _internal_metadata_),\n");
+  } else {
+    printer->Print(vars,
+    "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET("
+    "$classname$, _arena_),\n");
+  }
+
+  // is_default_instance_ offset.
+  if (HasFieldPresence(descriptor_->file())) {
+    printer->Print(vars,
+    "    -1);\n");
   } else {
     printer->Print(vars,
     "    GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET("
-    "$classname$, _arena_));\n");
+    "$classname$, _is_default_instance_));\n");
   }
 
   // Handle nested types.
@@ -1612,6 +1638,11 @@ GenerateSharedConstructorCode(io::Printer* printer) {
     "classname", classname_);
   printer->Indent();
 
+  if (!HasFieldPresence(descriptor_->file())) {
+    printer->Print(
+      "  _is_default_instance_ = false;\n");
+  }
+
   printer->Print(StrCat(
       uses_string_ ? "::google::protobuf::internal::GetEmptyString();\n" : "",
       "_cached_size_ = 0;\n").c_str());
@@ -1809,6 +1840,11 @@ GenerateStructors(io::Printer* printer) {
     "void $classname$::InitAsDefaultInstance() {\n",
     "classname", classname_);
 
+  if (!HasFieldPresence(descriptor_->file())) {
+    printer->Print(
+      "  _is_default_instance_ = true;\n");
+  }
+
   // The default instance needs all of its embedded message pointers
   // cross-linked to other default instances.  We can't do this initialization
   // in the constructor because some other default instances may not have been

src/google/protobuf/compiler/plugin.pb.cc

@@ -56,7 +56,8 @@ void protobuf_AssignDesc_google_2fprotobuf_2fcompiler_2fplugin_2eproto() {
       -1,
       -1,
       sizeof(CodeGeneratorRequest),
-      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorRequest, _internal_metadata_));
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorRequest, _internal_metadata_),
+      -1);
   CodeGeneratorResponse_descriptor_ = file->message_type(1);
   static const int CodeGeneratorResponse_offsets_[2] = {
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse, error_),
@@ -71,7 +72,8 @@ void protobuf_AssignDesc_google_2fprotobuf_2fcompiler_2fplugin_2eproto() {
       -1,
       -1,
       sizeof(CodeGeneratorResponse),
-      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse, _internal_metadata_));
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse, _internal_metadata_),
+      -1);
   CodeGeneratorResponse_File_descriptor_ = CodeGeneratorResponse_descriptor_->nested_type(0);
   static const int CodeGeneratorResponse_File_offsets_[3] = {
     GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse_File, name_),
@@ -87,7 +89,8 @@ void protobuf_AssignDesc_google_2fprotobuf_2fcompiler_2fplugin_2eproto() {
       -1,
       -1,
       sizeof(CodeGeneratorResponse_File),
-      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse_File, _internal_metadata_));
+      GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET(CodeGeneratorResponse_File, _internal_metadata_),
+      -1);
 }
 
 namespace {

src/google/protobuf/dynamic_message.cc

@@ -218,6 +218,7 @@ class DynamicMessage : public Message {
     int oneof_case_offset;
     int unknown_fields_offset;
     int extensions_offset;
+    int is_default_instance_offset;
 
     // Not owned by the TypeInfo.
     DynamicMessageFactory* factory;  // The factory that created this object.
@@ -316,6 +317,11 @@ void DynamicMessage::SharedCtor() {
         uint32(0);
   }
 
+  if (type_info_->is_default_instance_offset != -1) {
+    *reinterpret_cast<bool*>(
+        OffsetToPointer(type_info_->is_default_instance_offset)) = false;
+  }
+
   new(OffsetToPointer(type_info_->unknown_fields_offset)) UnknownFieldSet;
 
   if (type_info_->extensions_offset != -1) {
@@ -532,6 +538,14 @@ void DynamicMessage::CrossLinkPrototypes() {
         factory->GetPrototypeNoLock(field->message_type());
     }
   }
+
+  // Set as the default instance -- this affects field-presence semantics for
+  // proto3.
+  if (type_info_->is_default_instance_offset != -1) {
+    void* is_default_instance_ptr =
+        OffsetToPointer(type_info_->is_default_instance_offset);
+    *reinterpret_cast<bool*>(is_default_instance_ptr) = true;
+  }
 }
 
 Message* DynamicMessage::New() const {
@@ -641,11 +655,24 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
   size = AlignOffset(size);
 
   // Next the has_bits, which is an array of uint32s.
-  type_info->has_bits_offset = size;
-  int has_bits_array_size =
-    DivideRoundingUp(type->field_count(), bitsizeof(uint32));
-  size += has_bits_array_size * sizeof(uint32);
-  size = AlignOffset(size);
+  if (type->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) {
+    type_info->has_bits_offset = -1;
+  } else {
+    type_info->has_bits_offset = size;
+    int has_bits_array_size =
+      DivideRoundingUp(type->field_count(), bitsizeof(uint32));
+    size += has_bits_array_size * sizeof(uint32);
+    size = AlignOffset(size);
+  }
+
+  // The is_default_instance member, if any.
+  if (type->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) {
+    type_info->is_default_instance_offset = size;
+    size += sizeof(bool);
+    size = AlignOffset(size);
+  } else {
+    type_info->is_default_instance_offset = -1;
+  }
 
   // The oneof_case, if any. It is an array of uint32s.
   if (type->oneof_decl_count() > 0) {
@@ -731,7 +758,8 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
             type_info->pool,
             this,
             type_info->size,
-            -1 /* arena_offset */));
+            -1 /* arena_offset */,
+            type_info->is_default_instance_offset));
   } else {
     type_info->reflection.reset(
         new GeneratedMessageReflection(
@@ -744,7 +772,8 @@ const Message* DynamicMessageFactory::GetPrototypeNoLock(
             type_info->pool,
             this,
             type_info->size,
-            -1 /* arena_offset */));
+            -1 /* arena_offset */,
+            type_info->is_default_instance_offset));
   }
   // Cross link prototypes.
   prototype->CrossLinkPrototypes();

src/google/protobuf/dynamic_message_unittest.cc

@@ -46,6 +46,7 @@
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/test_util.h>
 #include <google/protobuf/unittest.pb.h>
+#include <google/protobuf/unittest_no_field_presence.pb.h>
 
 #include <google/protobuf/testing/googletest.h>
 #include <gtest/gtest.h>
@@ -65,6 +66,8 @@ class DynamicMessageTest : public testing::Test {
   const Message* packed_prototype_;
   const Descriptor* oneof_descriptor_;
   const Message* oneof_prototype_;
+  const Descriptor* proto3_descriptor_;
+  const Message* proto3_prototype_;
 
   DynamicMessageTest(): factory_(&pool_) {}
 
@@ -76,16 +79,20 @@ class DynamicMessageTest : public testing::Test {
     FileDescriptorProto unittest_file;
     FileDescriptorProto unittest_import_file;
     FileDescriptorProto unittest_import_public_file;
+    FileDescriptorProto unittest_no_field_presence_file;
 
     unittest::TestAllTypes::descriptor()->file()->CopyTo(&unittest_file);
     unittest_import::ImportMessage::descriptor()->file()->CopyTo(
       &unittest_import_file);
     unittest_import::PublicImportMessage::descriptor()->file()->CopyTo(
       &unittest_import_public_file);
+    proto2_nofieldpresence_unittest::TestAllTypes::descriptor()->
+        file()->CopyTo(&unittest_no_field_presence_file);
 
     ASSERT_TRUE(pool_.BuildFile(unittest_import_public_file) != NULL);
     ASSERT_TRUE(pool_.BuildFile(unittest_import_file) != NULL);
     ASSERT_TRUE(pool_.BuildFile(unittest_file) != NULL);
+    ASSERT_TRUE(pool_.BuildFile(unittest_no_field_presence_file) != NULL);
 
     descriptor_ = pool_.FindMessageTypeByName("protobuf_unittest.TestAllTypes");
     ASSERT_TRUE(descriptor_ != NULL);
@@ -105,6 +112,12 @@ class DynamicMessageTest : public testing::Test {
       pool_.FindMessageTypeByName("protobuf_unittest.TestOneof2");
     ASSERT_TRUE(oneof_descriptor_ != NULL);
     oneof_prototype_ = factory_.GetPrototype(oneof_descriptor_);
+
+    proto3_descriptor_ =
+      pool_.FindMessageTypeByName(
+          "proto2_nofieldpresence_unittest.TestAllTypes");
+    ASSERT_TRUE(proto3_descriptor_ != NULL);
+    proto3_prototype_ = factory_.GetPrototype(proto3_descriptor_);
   }
 };
 
@@ -233,6 +246,40 @@ TEST_F(DynamicMessageTest, Arena) {
   // Return without freeing: should not leak.
 }
 
+TEST_F(DynamicMessageTest, Proto3) {
+  Message* message = proto3_prototype_->New();
+  const Reflection* refl = message->GetReflection();
+  const Descriptor* desc = message->GetDescriptor();
+
+  // Just test a single primtive and single message field here to make sure we
+  // are getting the no-field-presence semantics elsewhere. DynamicMessage uses
+  // GeneratedMessageReflection under the hood, so the rest should be fine as
+  // long as GMR recognizes that we're using a proto3 message.
+  const FieldDescriptor* optional_int32 =
+      desc->FindFieldByName("optional_int32");
+  const FieldDescriptor* optional_msg =
+      desc->FindFieldByName("optional_nested_message");
+  EXPECT_TRUE(optional_int32 != NULL);
+  EXPECT_TRUE(optional_msg != NULL);
+
+  EXPECT_EQ(false, refl->HasField(*message, optional_int32));
+  refl->SetInt32(message, optional_int32, 42);
+  EXPECT_EQ(true, refl->HasField(*message, optional_int32));
+  refl->SetInt32(message, optional_int32, 0);
+  EXPECT_EQ(false, refl->HasField(*message, optional_int32));
+
+  EXPECT_EQ(false, refl->HasField(*message, optional_msg));
+  refl->MutableMessage(message, optional_msg);
+  EXPECT_EQ(true, refl->HasField(*message, optional_msg));
+  delete refl->ReleaseMessage(message, optional_msg);
+  EXPECT_EQ(false, refl->HasField(*message, optional_msg));
+
+  // Also ensure that the default instance handles field presence properly.
+  EXPECT_EQ(false, refl->HasField(*proto3_prototype_, optional_msg));
+
+  delete message;
+}
+
 
 }  // namespace protobuf
 }  // namespace google

src/google/protobuf/generated_message_reflection.cc

@@ -193,7 +193,8 @@ GeneratedMessageReflection::GeneratedMessageReflection(
     const DescriptorPool* descriptor_pool,
     MessageFactory* factory,
     int object_size,
-    int arena_offset)
+    int arena_offset,
+    int is_default_instance_offset)
   : descriptor_       (descriptor),
     default_instance_ (default_instance),
     offsets_          (offsets),
@@ -201,6 +202,7 @@ GeneratedMessageReflection::GeneratedMessageReflection(
     unknown_fields_offset_(unknown_fields_offset),
     extensions_offset_(extensions_offset),
     arena_offset_     (arena_offset),
+    is_default_instance_offset_(is_default_instance_offset),
     object_size_      (object_size),
     descriptor_pool_  ((descriptor_pool == NULL) ?
                          DescriptorPool::generated_pool() :
@@ -220,7 +222,8 @@ GeneratedMessageReflection::GeneratedMessageReflection(
     const DescriptorPool* descriptor_pool,
     MessageFactory* factory,
     int object_size,
-    int arena_offset)
+    int arena_offset,
+    int is_default_instance_offset)
   : descriptor_       (descriptor),
     default_instance_ (default_instance),
     default_oneof_instance_ (default_oneof_instance),
@@ -230,6 +233,7 @@ GeneratedMessageReflection::GeneratedMessageReflection(
     unknown_fields_offset_(unknown_fields_offset),
     extensions_offset_(extensions_offset),
     arena_offset_     (arena_offset),
+    is_default_instance_offset_(is_default_instance_offset),
     object_size_      (object_size),
     descriptor_pool_  ((descriptor_pool == NULL) ?
                          DescriptorPool::generated_pool() :
@@ -1829,6 +1833,17 @@ GeneratedMessageReflection::MutableInternalMetadataWithArena(
   return reinterpret_cast<InternalMetadataWithArena*>(ptr);
 }
 
+inline bool
+GeneratedMessageReflection::GetIsDefaultInstance(
+    const Message& message) const {
+  if (is_default_instance_offset_ == kHasNoDefaultInstanceField) {
+    return false;
+  }
+  const void* ptr = reinterpret_cast<const uint8*>(&message) +
+      is_default_instance_offset_;
+  return *reinterpret_cast<const bool*>(ptr);
+}
+
 // Simple accessors for manipulating has_bits_.
 inline bool GeneratedMessageReflection::HasBit(
     const Message& message, const FieldDescriptor* field) const {
@@ -1836,7 +1851,8 @@ inline bool GeneratedMessageReflection::HasBit(
     // proto3: no has-bits. All fields present except messages, which are
     // present only if their message-field pointer is non-NULL.
     if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
-      return GetRaw<const Message*>(message, field) != NULL;
+      return !GetIsDefaultInstance(message) &&
+          GetRaw<const Message*>(message, field) != NULL;
     } else {
       // Non-message field (and non-oneof, since that was handled in HasField()
       // before calling us), and singular (again, checked in HasField). So, this
@@ -2082,7 +2098,8 @@ GeneratedMessageReflection::NewGeneratedMessageReflection(
     const void* default_oneof_instance,
     int oneof_case_offset,
     int object_size,
-    int arena_offset) {
+    int arena_offset,
+    int is_default_instance_offset) {
   return new GeneratedMessageReflection(descriptor,
                                         default_instance,
                                         offsets,
@@ -2094,7 +2111,8 @@ GeneratedMessageReflection::NewGeneratedMessageReflection(
                                         DescriptorPool::generated_pool(),
                                         MessageFactory::generated_factory(),
                                         object_size,
-                                        arena_offset);
+                                        arena_offset,
+                                        is_default_instance_offset);
 }
 
 GeneratedMessageReflection*
@@ -2106,7 +2124,8 @@ GeneratedMessageReflection::NewGeneratedMessageReflection(
     int unknown_fields_offset,
     int extensions_offset,
     int object_size,
-    int arena_offset) {
+    int arena_offset,
+    int is_default_instance_offset) {
   return new GeneratedMessageReflection(descriptor,
                                         default_instance,
                                         offsets,
@@ -2116,7 +2135,8 @@ GeneratedMessageReflection::NewGeneratedMessageReflection(
                                         DescriptorPool::generated_pool(),
                                         MessageFactory::generated_factory(),
                                         object_size,
-                                        arena_offset);
+                                        arena_offset,
+                                        is_default_instance_offset);
 }
 
 }  // namespace internal

src/google/protobuf/generated_message_reflection.h

@@ -136,7 +136,8 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
                              const DescriptorPool* pool,
                              MessageFactory* factory,
                              int object_size,
-                             int arena_offset);
+                             int arena_offset,
+                             int is_default_instance_offset = -1);
 
   // Similar with the construction above. Call this construction if the
   // message has oneof definition.
@@ -173,7 +174,8 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
                              const DescriptorPool* pool,
                              MessageFactory* factory,
                              int object_size,
-                             int arena_offset);
+                             int arena_offset,
+                             int is_default_instance_offset = -1);
   ~GeneratedMessageReflection();
 
   // Shorter-to-call helpers for the above two constructions that work if the
@@ -190,7 +192,8 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
       const void* default_oneof_instance,
       int oneof_case_offset,
       int object_size,
-      int arena_offset);
+      int arena_offset,
+      int is_default_instance_offset = -1);
 
   static GeneratedMessageReflection* NewGeneratedMessageReflection(
       const Descriptor* descriptor,
@@ -200,7 +203,8 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
       int unknown_fields_offset,
       int extensions_offset,
       int object_size,
-      int arena_offset);
+      int arena_offset,
+      int is_default_instance_offset = -1);
 
   // implements Reflection -------------------------------------------
 
@@ -414,8 +418,11 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
   int unknown_fields_offset_;
   int extensions_offset_;
   int arena_offset_;
+  int is_default_instance_offset_;
   int object_size_;
 
+  static const int kHasNoDefaultInstanceField = -1;
+
   const DescriptorPool* descriptor_pool_;
   MessageFactory* message_factory_;
 
@@ -446,6 +453,8 @@ class LIBPROTOBUF_EXPORT GeneratedMessageReflection : public Reflection {
   inline internal::InternalMetadataWithArena*
       MutableInternalMetadataWithArena(Message* message) const;
 
+  inline bool GetIsDefaultInstance(const Message& message) const;
+
   inline bool HasBit(const Message& message,
                      const FieldDescriptor* field) const;
   inline void SetBit(Message* message,

src/google/protobuf/no_field_presence_test.cc

@@ -269,6 +269,10 @@ TEST(NoFieldPresenceTest, MessageFieldPresenceTest) {
   EXPECT_EQ(true, message.has_optional_lazy_message());
   message.clear_optional_lazy_message();
   EXPECT_EQ(false, message.has_optional_lazy_message());
+
+  // Test field presence of a message field on the default instance.
+  EXPECT_EQ(false, proto2_nofieldpresence_unittest::TestAllTypes::
+            default_instance().has_optional_nested_message());
 }
 
 TEST(NoFieldPresenceTest, ReflectionHasFieldTest) {
@@ -287,6 +291,13 @@ TEST(NoFieldPresenceTest, ReflectionHasFieldTest) {
     EXPECT_EQ(false, r->HasField(message, field));
   }
 
+  // Test field presence of a message field on the default instance.
+  const google::protobuf::FieldDescriptor* msg_field =
+      desc->FindFieldByName("optional_nested_message");
+  EXPECT_EQ(false, r->HasField(
+      proto2_nofieldpresence_unittest::TestAllTypes::
+      default_instance(), msg_field));
+
   // Fill all fields, expect everything to report true (check oneofs below).
   FillValues(&message);
   for (int i = 0; i < desc->field_count(); i++) {