Adds support for proto2 syntax for Ruby gem.

This change only adds basic proto2 support without advanced features
like extensions, custom options, maps, etc.

The protoc binary now generates ruby code for proto2 syntax.
However, for now, it is restricted to proto2 files without advanced features
like extensions, in which case it still errors out.

This change also modifies the DSL to add proto messages to the DescriptorPool.
There is a new DSL Builder#add_file to create a new FileDescriptor. With this,
the generated ruby DSL looks something like:

Google::Protobuf::DescriptorPool.generated_pool.build do
  add_file "test.proto" do
    add_message "foo" do
      optional :val, :int32, 1
    end
  end
end

.gitignore

@@ -171,9 +171,14 @@ js/testproto_libs2.js
 
 # ruby test output
 ruby/lib/
+ruby/tests/basic_test_pb.rb
+ruby/tests/basic_test_proto2_pb.rb
 ruby/tests/generated_code_pb.rb
 ruby/tests/test_import_pb.rb
 ruby/tests/test_ruby_package_pb.rb
+ruby/tests/generated_code_proto2_pb.rb
+ruby/tests/test_import_proto2_pb.rb
+ruby/tests/test_ruby_package_proto2_pb.rb
 ruby/Gemfile.lock
 ruby/compatibility_tests/v3.0.0/protoc
 ruby/compatibility_tests/v3.0.0/tests/generated_code_pb.rb

ruby/Rakefile

@@ -86,20 +86,45 @@ end
 
 # Proto for tests.
 genproto_output << "tests/generated_code.rb"
+genproto_output << "tests/generated_code_proto2.rb"
 genproto_output << "tests/test_import.rb"
+genproto_output << "tests/test_import_proto2.rb"
 genproto_output << "tests/test_ruby_package.rb"
+genproto_output << "tests/test_ruby_package_proto2.rb"
+genproto_output << "tests/basic_test.rb"
+genproto_output << "tests/basic_test_proto2.rb"
 file "tests/generated_code.rb" => "tests/generated_code.proto" do |file_task|
   sh "../src/protoc --ruby_out=. tests/generated_code.proto"
 end
 
+file "tests/generated_code_proto2.rb" => "tests/generated_code_proto2.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/generated_code_proto2.proto"
+end
+
 file "tests/test_import.rb" => "tests/test_import.proto" do |file_task|
   sh "../src/protoc --ruby_out=. tests/test_import.proto"
 end
 
+file "tests/test_import_proto2.rb" => "tests/test_import_proto2.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/test_import_proto2.proto"
+end
+
 file "tests/test_ruby_package.rb" => "tests/test_ruby_package.proto" do |file_task|
   sh "../src/protoc --ruby_out=. tests/test_ruby_package.proto"
 end
 
+file "tests/test_ruby_package_proto2.rb" => "tests/test_ruby_package_proto2.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/test_ruby_package_proto2.proto"
+end
+
+file "tests/basic_test.rb" => "tests/basic_test.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/basic_test.proto"
+end
+
+file "tests/basic_test_proto2.rb" => "tests/basic_test_proto2.proto" do |file_task|
+  sh "../src/protoc --ruby_out=. tests/basic_test_proto2.proto"
+end
+
 task :genproto => genproto_output
 
 task :clean do
@@ -110,7 +135,7 @@ Gem::PackageTask.new(spec) do |pkg|
 end
 
 Rake::TestTask.new(:test => :build) do |t|
-  t.test_files = FileList["tests/*.rb"].exclude("tests/gc_test.rb")
+  t.test_files = FileList["tests/*.rb"].exclude("tests/gc_test.rb", "tests/common_tests.rb")
 end
 
 # gc_test needs to be split out to ensure the generated file hasn't been

ruby/ext/google/protobuf_c/encode_decode.c

@@ -100,24 +100,34 @@ void stringsink_uninit(stringsink *sink) {
 
 #define DEREF(msg, ofs, type) *(type*)(((uint8_t *)msg) + ofs)
 
-// Creates a handlerdata that simply contains the offset for this field.
-static const void* newhandlerdata(upb_handlers* h, uint32_t ofs) {
-  size_t* hd_ofs = ALLOC(size_t);
-  *hd_ofs = ofs;
-  upb_handlers_addcleanup(h, hd_ofs, xfree);
-  return hd_ofs;
+typedef struct {
+  size_t ofs;
+  int32_t hasbit;
+} field_handlerdata_t;
+
+// Creates a handlerdata that contains the offset and the hasbit for the field
+static const void* newhandlerdata(upb_handlers* h, uint32_t ofs, int32_t hasbit) {
+  field_handlerdata_t *hd = ALLOC(field_handlerdata_t);
+  hd->ofs = ofs;
+  hd->hasbit = hasbit;
+  upb_handlers_addcleanup(h, hd, xfree);
+  return hd;
 }
 
 typedef struct {
   size_t ofs;
+  int32_t hasbit;
   const upb_msgdef *md;
 } submsg_handlerdata_t;
 
 // Creates a handlerdata that contains offset and submessage type information.
-static const void *newsubmsghandlerdata(upb_handlers* h, uint32_t ofs,
+static const void *newsubmsghandlerdata(upb_handlers* h,
+                                        uint32_t ofs,
+                                        int32_t hasbit,
                                         const upb_fielddef* f) {
   submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t);
   hd->ofs = ofs;
+  hd->hasbit = hasbit;
   hd->md = upb_fielddef_msgsubdef(f);
   upb_handlers_addcleanup(h, hd, xfree);
   return hd;
@@ -189,6 +199,13 @@ static void* appendstr_handler(void *closure,
   return (void*)str;
 }
 
+static void set_hasbit(void *closure, int32_t hasbit) {
+  if (hasbit > 0) {
+    uint8_t* storage = closure;
+    storage[hasbit/8] |= 1 << (hasbit % 8);
+  }
+}
+
 // Appends a 'bytes' string to a repeated field.
 static void* appendbytes_handler(void *closure,
                                  const void *hd,
@@ -205,10 +222,12 @@ static void* str_handler(void *closure,
                          const void *hd,
                          size_t size_hint) {
   MessageHeader* msg = closure;
-  const size_t *ofs = hd;
+  const field_handlerdata_t *fieldhandler = hd;
+
   VALUE str = rb_str_new2("");
   rb_enc_associate(str, kRubyStringUtf8Encoding);
-  DEREF(msg, *ofs, VALUE) = str;
+  DEREF(msg, fieldhandler->ofs, VALUE) = str;
+  set_hasbit(closure, fieldhandler->hasbit);
   return (void*)str;
 }
 
@@ -217,10 +236,12 @@ static void* bytes_handler(void *closure,
                            const void *hd,
                            size_t size_hint) {
   MessageHeader* msg = closure;
-  const size_t *ofs = hd;
+  const field_handlerdata_t *fieldhandler = hd;
+
   VALUE str = rb_str_new2("");
   rb_enc_associate(str, kRubyString8bitEncoding);
-  DEREF(msg, *ofs, VALUE) = str;
+  DEREF(msg, fieldhandler->ofs, VALUE) = str;
+  set_hasbit(closure, fieldhandler->hasbit);
   return (void*)str;
 }
 
@@ -280,8 +301,11 @@ static void *submsg_handler(void *closure, const void *hd) {
         rb_class_new_instance(0, NULL, subklass);
   }
 
+  set_hasbit(closure, submsgdata->hasbit);
+
   submsg_rb = DEREF(msg, submsgdata->ofs, VALUE);
   TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg);
+
   return submsg;
 }
 
@@ -500,7 +524,7 @@ static void add_handlers_for_repeated_field(upb_handlers *h,
                                             const upb_fielddef *f,
                                             size_t offset) {
   upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-  upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset));
+  upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, -1));
   upb_handlers_setstartseq(h, f, startseq_handler, &attr);
   upb_handlerattr_uninit(&attr);
 
@@ -534,7 +558,7 @@ static void add_handlers_for_repeated_field(upb_handlers *h,
     }
     case UPB_TYPE_MESSAGE: {
       upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, 0, f));
+      upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, 0, -1, f));
       upb_handlers_setstartsubmsg(h, f, appendsubmsg_handler, &attr);
       upb_handlerattr_uninit(&attr);
       break;
@@ -545,7 +569,15 @@ static void add_handlers_for_repeated_field(upb_handlers *h,
 // Set up handlers for a singular field.
 static void add_handlers_for_singular_field(upb_handlers *h,
                                             const upb_fielddef *f,
-                                            size_t offset) {
+                                            size_t offset,
+                                            size_t hasbit_off) {
+  // The offset we pass to UPB points to the start of the Message,
+  // rather than the start of where our data is stored.
+  int32_t hasbit = -1;
+  if (hasbit_off != MESSAGE_FIELD_NO_HASBIT) {
+    hasbit = hasbit_off + sizeof(MessageHeader) * 8;
+  }
+
   switch (upb_fielddef_type(f)) {
     case UPB_TYPE_BOOL:
     case UPB_TYPE_INT32:
@@ -555,13 +587,13 @@ static void add_handlers_for_singular_field(upb_handlers *h,
     case UPB_TYPE_INT64:
     case UPB_TYPE_UINT64:
     case UPB_TYPE_DOUBLE:
-      upb_msg_setscalarhandler(h, f, offset, -1);
+      upb_msg_setscalarhandler(h, f, offset, hasbit);
       break;
     case UPB_TYPE_STRING:
     case UPB_TYPE_BYTES: {
       bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES;
       upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset));
+      upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset, hasbit));
       upb_handlers_setstartstr(h, f,
                                is_bytes ? bytes_handler : str_handler,
                                &attr);
@@ -572,7 +604,9 @@ static void add_handlers_for_singular_field(upb_handlers *h,
     }
     case UPB_TYPE_MESSAGE: {
       upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
-      upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, offset, f));
+      upb_handlerattr_sethandlerdata(&attr,
+				     newsubmsghandlerdata(h, offset,
+							  hasbit, f));
       upb_handlers_setstartsubmsg(h, f, submsg_handler, &attr);
       upb_handlerattr_uninit(&attr);
       break;
@@ -610,10 +644,12 @@ static void add_handlers_for_mapentry(const upb_msgdef* msgdef,
 
   add_handlers_for_singular_field(
       h, key_field,
-      offsetof(map_parse_frame_t, key_storage));
+      offsetof(map_parse_frame_t, key_storage),
+      MESSAGE_FIELD_NO_HASBIT);
   add_handlers_for_singular_field(
       h, value_field,
-      offsetof(map_parse_frame_t, value_storage));
+      offsetof(map_parse_frame_t, value_storage),
+      MESSAGE_FIELD_NO_HASBIT);
 }
 
 // Set up handlers for a oneof field.
@@ -718,7 +754,8 @@ static void add_handlers_for_message(const void *closure, upb_handlers *h) {
     } else if (upb_fielddef_isseq(f)) {
       add_handlers_for_repeated_field(h, f, offset);
     } else {
-      add_handlers_for_singular_field(h, f, offset);
+      add_handlers_for_singular_field(
+          h, f, offset, desc->layout->fields[upb_fielddef_index(f)].hasbit);
     }
   }
 }
@@ -901,11 +938,6 @@ VALUE Message_decode_json(VALUE klass, VALUE data) {
 
 /* msgvisitor *****************************************************************/
 
-// TODO: If/when we support proto2 semantics in addition to the current proto3
-// semantics, which means that we have true field presence, we will want to
-// modify msgvisitor so that it emits all present fields rather than all
-// non-default-value fields.
-
 static void putmsg(VALUE msg, const Descriptor* desc,
                    upb_sink *sink, int depth, bool emit_defaults);
 
@@ -962,6 +994,7 @@ static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
   int size;
 
   if (ary == Qnil) return;
+  if (!emit_defaults && NUM2INT(RepeatedField_length(ary)) == 0) return;
 
   size = NUM2INT(RepeatedField_length(ary));
   if (size == 0 && !emit_defaults) return;
@@ -1062,6 +1095,8 @@ static void putmap(VALUE map, const upb_fielddef *f, upb_sink *sink,
   Map_iter it;
 
   if (map == Qnil) return;
+  if (!emit_defaults && Map_length(map) == 0) return;
+
   self = ruby_to_Map(map);
 
   upb_sink_startseq(sink, getsel(f, UPB_HANDLER_STARTSEQ), &subsink);
@@ -1144,7 +1179,15 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
       }
     } else if (upb_fielddef_isstring(f)) {
       VALUE str = DEREF(msg, offset, VALUE);
-      if (is_matching_oneof || emit_defaults || RSTRING_LEN(str) > 0) {
+      bool is_default = false;
+
+      if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO2) {
+        is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse;
+      } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {
+        is_default = RSTRING_LEN(str) == 0;
+      }
+
+      if (is_matching_oneof || emit_defaults || !is_default) {
         putstr(str, f, sink);
       }
     } else if (upb_fielddef_issubmsg(f)) {
@@ -1155,7 +1198,13 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
 #define T(upbtypeconst, upbtype, ctype, default_value)                          \
   case upbtypeconst: {                                                          \
       ctype value = DEREF(msg, offset, ctype);                                  \
-      if (is_matching_oneof || emit_defaults || value != default_value) { \
+      bool is_default = false;                                                  \
+      if (upb_fielddef_haspresence(f)) {                                        \
+        is_default = layout_has(desc->layout, Message_data(msg), f) == Qfalse;  \
+      } else if (upb_msgdef_syntax(desc->msgdef) == UPB_SYNTAX_PROTO3) {        \
+        is_default = default_value == value;                                    \
+      }                                                                         \
+      if (is_matching_oneof || emit_defaults || !is_default) {                  \
         upb_sink_put##upbtype(sink, sel, value);                                \
       }                                                                         \
     }                                                                           \

ruby/ext/google/protobuf_c/protobuf.c

@@ -91,12 +91,14 @@ void Init_protobuf_c() {
   descriptor_instancevar_interned = rb_intern(kDescriptorInstanceVar);
   DescriptorPool_register(protobuf);
   Descriptor_register(protobuf);
+  FileDescriptor_register(protobuf);
   FieldDescriptor_register(protobuf);
   OneofDescriptor_register(protobuf);
   EnumDescriptor_register(protobuf);
   MessageBuilderContext_register(internal);
   OneofBuilderContext_register(internal);
   EnumBuilderContext_register(internal);
+  FileBuilderContext_register(internal);
   Builder_register(internal);
   RepeatedField_register(protobuf);
   Map_register(protobuf);

ruby/ext/google/protobuf_c/protobuf.h

@@ -40,6 +40,7 @@
 // Forward decls.
 struct DescriptorPool;
 struct Descriptor;
+struct FileDescriptor;
 struct FieldDescriptor;
 struct EnumDescriptor;
 struct MessageLayout;
@@ -47,10 +48,12 @@ struct MessageField;
 struct MessageHeader;
 struct MessageBuilderContext;
 struct EnumBuilderContext;
+struct FileBuilderContext;
 struct Builder;
 
 typedef struct DescriptorPool DescriptorPool;
 typedef struct Descriptor Descriptor;
+typedef struct FileDescriptor FileDescriptor;
 typedef struct FieldDescriptor FieldDescriptor;
 typedef struct OneofDescriptor OneofDescriptor;
 typedef struct EnumDescriptor EnumDescriptor;
@@ -60,6 +63,7 @@ typedef struct MessageHeader MessageHeader;
 typedef struct MessageBuilderContext MessageBuilderContext;
 typedef struct OneofBuilderContext OneofBuilderContext;
 typedef struct EnumBuilderContext EnumBuilderContext;
+typedef struct FileBuilderContext FileBuilderContext;
 typedef struct Builder Builder;
 
 /*
@@ -118,6 +122,10 @@ struct Descriptor {
   const upb_handlers* json_serialize_handlers_preserve;
 };
 
+struct FileDescriptor {
+  const upb_filedef* filedef;
+};
+
 struct FieldDescriptor {
   const upb_fielddef* fielddef;
 };
@@ -145,18 +153,27 @@ struct EnumBuilderContext {
   VALUE enumdesc;
 };
 
+struct FileBuilderContext {
+  VALUE pending_list;
+  VALUE file_descriptor;
+  VALUE builder;
+};
+
 struct Builder {
   VALUE pending_list;
+  VALUE default_file_descriptor;
   upb_def** defs;  // used only while finalizing
 };
 
 extern VALUE cDescriptorPool;
 extern VALUE cDescriptor;
+extern VALUE cFileDescriptor;
 extern VALUE cFieldDescriptor;
 extern VALUE cEnumDescriptor;
 extern VALUE cMessageBuilderContext;
 extern VALUE cOneofBuilderContext;
 extern VALUE cEnumBuilderContext;
+extern VALUE cFileBuilderContext;
 extern VALUE cBuilder;
 
 extern VALUE cError;
@@ -175,7 +192,7 @@ VALUE DescriptorPool_alloc(VALUE klass);
 void DescriptorPool_register(VALUE module);
 DescriptorPool* ruby_to_DescriptorPool(VALUE value);
 VALUE DescriptorPool_add(VALUE _self, VALUE def);
-VALUE DescriptorPool_build(VALUE _self);
+VALUE DescriptorPool_build(int argc, VALUE* argv, VALUE _self);
 VALUE DescriptorPool_lookup(VALUE _self, VALUE name);
 VALUE DescriptorPool_generated_pool(VALUE _self);
 
@@ -184,6 +201,7 @@ void Descriptor_free(void* _self);
 VALUE Descriptor_alloc(VALUE klass);
 void Descriptor_register(VALUE module);
 Descriptor* ruby_to_Descriptor(VALUE value);
+VALUE Descriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
 VALUE Descriptor_name(VALUE _self);
 VALUE Descriptor_name_set(VALUE _self, VALUE str);
 VALUE Descriptor_each(VALUE _self);
@@ -193,8 +211,19 @@ VALUE Descriptor_add_oneof(VALUE _self, VALUE obj);
 VALUE Descriptor_each_oneof(VALUE _self);
 VALUE Descriptor_lookup_oneof(VALUE _self, VALUE name);
 VALUE Descriptor_msgclass(VALUE _self);
+VALUE Descriptor_file_descriptor(VALUE _self);
 extern const rb_data_type_t _Descriptor_type;
 
+void FileDescriptor_mark(void* _self);
+void FileDescriptor_free(void* _self);
+VALUE FileDescriptor_alloc(VALUE klass);
+void FileDescriptor_register(VALUE module);
+FileDescriptor* ruby_to_FileDescriptor(VALUE value);
+VALUE FileDescriptor_initialize(int argc, VALUE* argv, VALUE _self);
+VALUE FileDescriptor_name(VALUE _self);
+VALUE FileDescriptor_syntax(VALUE _self);
+VALUE FileDescriptor_syntax_set(VALUE _self, VALUE syntax);
+
 void FieldDescriptor_mark(void* _self);
 void FieldDescriptor_free(void* _self);
 VALUE FieldDescriptor_alloc(VALUE klass);
@@ -204,6 +233,8 @@ VALUE FieldDescriptor_name(VALUE _self);
 VALUE FieldDescriptor_name_set(VALUE _self, VALUE str);
 VALUE FieldDescriptor_type(VALUE _self);
 VALUE FieldDescriptor_type_set(VALUE _self, VALUE type);
+VALUE FieldDescriptor_default(VALUE _self);
+VALUE FieldDescriptor_default_set(VALUE _self, VALUE default_value);
 VALUE FieldDescriptor_label(VALUE _self);
 VALUE FieldDescriptor_label_set(VALUE _self, VALUE label);
 VALUE FieldDescriptor_number(VALUE _self);
@@ -211,6 +242,8 @@ VALUE FieldDescriptor_number_set(VALUE _self, VALUE number);
 VALUE FieldDescriptor_submsg_name(VALUE _self);
 VALUE FieldDescriptor_submsg_name_set(VALUE _self, VALUE value);
 VALUE FieldDescriptor_subtype(VALUE _self);
+VALUE FieldDescriptor_has(VALUE _self, VALUE msg_rb);
+VALUE FieldDescriptor_clear(VALUE _self, VALUE msg_rb);
 VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb);
 VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value);
 upb_fieldtype_t ruby_to_fieldtype(VALUE type);
@@ -231,6 +264,8 @@ void EnumDescriptor_free(void* _self);
 VALUE EnumDescriptor_alloc(VALUE klass);
 void EnumDescriptor_register(VALUE module);
 EnumDescriptor* ruby_to_EnumDescriptor(VALUE value);
+VALUE EnumDescriptor_initialize(VALUE _self, VALUE file_descriptor_rb);
+VALUE EnumDescriptor_file_descriptor(VALUE _self);
 VALUE EnumDescriptor_name(VALUE _self);
 VALUE EnumDescriptor_name_set(VALUE _self, VALUE str);
 VALUE EnumDescriptor_add_value(VALUE _self, VALUE name, VALUE number);
@@ -272,12 +307,23 @@ EnumBuilderContext* ruby_to_EnumBuilderContext(VALUE value);
 VALUE EnumBuilderContext_initialize(VALUE _self, VALUE enumdesc);
 VALUE EnumBuilderContext_value(VALUE _self, VALUE name, VALUE number);
 
+void FileBuilderContext_mark(void* _self);
+void FileBuilderContext_free(void* _self);
+VALUE FileBuilderContext_alloc(VALUE klass);
+void FileBuilderContext_register(VALUE module);
+VALUE FileBuilderContext_initialize(VALUE _self, VALUE file_descriptor,
+				    VALUE builder);
+VALUE FileBuilderContext_add_message(VALUE _self, VALUE name);
+VALUE FileBuilderContext_add_enum(VALUE _self, VALUE name);
+VALUE FileBuilderContext_pending_descriptors(VALUE _self);
+
 void Builder_mark(void* _self);
 void Builder_free(void* _self);
 VALUE Builder_alloc(VALUE klass);
 void Builder_register(VALUE module);
 Builder* ruby_to_Builder(VALUE value);
 VALUE Builder_initialize(VALUE _self);
+VALUE Builder_add_file(int argc, VALUE *argv, VALUE _self);
 VALUE Builder_add_message(VALUE _self, VALUE name);
 VALUE Builder_add_enum(VALUE _self, VALUE name);
 VALUE Builder_finalize_to_pool(VALUE _self, VALUE pool_rb);
@@ -443,10 +489,12 @@ VALUE Map_iter_value(Map_iter* iter);
 // -----------------------------------------------------------------------------
 
 #define MESSAGE_FIELD_NO_CASE ((size_t)-1)
+#define MESSAGE_FIELD_NO_HASBIT ((size_t)-1)
 
 struct MessageField {
   size_t offset;
   size_t case_offset;  // for oneofs, a uint32. Else, MESSAGE_FIELD_NO_CASE.
+  size_t hasbit;
 };
 
 struct MessageLayout {
@@ -457,6 +505,9 @@ struct MessageLayout {
 
 MessageLayout* create_layout(const upb_msgdef* msgdef);
 void free_layout(MessageLayout* layout);
+bool field_contains_hasbit(MessageLayout* layout,
+                 const upb_fielddef* field);
+VALUE layout_get_default(const upb_fielddef* field);
 VALUE layout_get(MessageLayout* layout,
                  const void* storage,
                  const upb_fielddef* field);
@@ -464,6 +515,12 @@ void layout_set(MessageLayout* layout,
                 void* storage,
                 const upb_fielddef* field,
                 VALUE val);
+VALUE layout_has(MessageLayout* layout,
+                 const void* storage,
+                 const upb_fielddef* field);
+void layout_clear(MessageLayout* layout,
+                 const void* storage,
+                 const upb_fielddef* field);
 void layout_init(MessageLayout* layout, void* storage);
 void layout_mark(MessageLayout* layout, void* storage);
 void layout_dup(MessageLayout* layout, void* to, void* from);

ruby/tests/basic_proto2.rb

+#!/usr/bin/ruby
+
+# basic_test_pb.rb is in the same directory as this test.
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__)))
+
+require 'basic_test_proto2_pb'
+require 'common_tests'
+require 'google/protobuf'
+require 'json'
+require 'test/unit'
+
+# ------------- generated code --------------
+
+module BasicTestProto2
+  pool = Google::Protobuf::DescriptorPool.new
+  pool.build do
+    add_file "test_proto2.proto", syntax: :proto2 do
+      add_message "BadFieldNames" do
+        optional :dup, :int32, 1
+        optional :class, :int32, 2
+        optional :"a.b", :int32, 3
+      end
+    end
+  end
+
+  BadFieldNames = pool.lookup("BadFieldNames").msgclass
+
+# ------------ test cases ---------------
+
+  class MessageContainerTest < Test::Unit::TestCase
+    # Required by CommonTests module to resolve proto2 proto classes used in tests.
+    def proto_module
+      ::BasicTestProto2
+    end
+    include CommonTests
+
+    def test_has_field
+      m = TestMessage.new
+      assert_false m.has_optional_int32?
+      assert_false TestMessage.descriptor.lookup('optional_int32').has?(m)
+      assert_false m.has_optional_int64?
+      assert_false TestMessage.descriptor.lookup('optional_int64').has?(m)
+      assert_false m.has_optional_uint32?
+      assert_false TestMessage.descriptor.lookup('optional_uint32').has?(m)
+      assert_false m.has_optional_uint64?
+      assert_false TestMessage.descriptor.lookup('optional_uint64').has?(m)
+      assert_false m.has_optional_bool?
+      assert_false TestMessage.descriptor.lookup('optional_bool').has?(m)
+      assert_false m.has_optional_float?
+      assert_false TestMessage.descriptor.lookup('optional_float').has?(m)
+      assert_false m.has_optional_double?
+      assert_false TestMessage.descriptor.lookup('optional_double').has?(m)
+      assert_false m.has_optional_string?
+      assert_false TestMessage.descriptor.lookup('optional_string').has?(m)
+      assert_false m.has_optional_bytes?
+      assert_false TestMessage.descriptor.lookup('optional_bytes').has?(m)
+      assert_false m.has_optional_enum?
+      assert_false TestMessage.descriptor.lookup('optional_enum').has?(m)
+
+      m = TestMessage.new(:optional_int32 => nil)
+      assert_false m.has_optional_int32?
+
+      assert_raise NoMethodError do
+        m.has_repeated_msg?
+      end
+      assert_raise ArgumentError do
+        TestMessage.descriptor.lookup('repeated_msg').has?(m)
+      end
+
+      m.optional_msg = TestMessage2.new
+      assert_true m.has_optional_msg?
+      assert_true TestMessage.descriptor.lookup('optional_msg').has?(m)
+
+      m = OneofMessage.new
+      assert_false m.has_my_oneof?
+      m.a = "foo"
+      assert_true m.has_a?
+      assert_true OneofMessage.descriptor.lookup('a').has?(m)
+      assert_equal "foo", m.a
+      assert_true m.has_my_oneof?
+      assert_false m.has_b?
+      assert_false OneofMessage.descriptor.lookup('b').has?(m)
+      assert_false m.has_c?
+      assert_false OneofMessage.descriptor.lookup('c').has?(m)
+      assert_false m.has_d?
+      assert_false OneofMessage.descriptor.lookup('d').has?(m)
+
+      m = OneofMessage.new
+      m.b = 100
+      assert_true m.has_b?
+      assert_equal 100, m.b
+      assert_true m.has_my_oneof?
+      assert_false m.has_a?
+      assert_false m.has_c?
+      assert_false m.has_d?
+
+      m = OneofMessage.new
+      m.c = TestMessage2.new
+      assert_true m.has_c?
+      assert_equal TestMessage2.new, m.c
+      assert_true m.has_my_oneof?
+      assert_false m.has_a?
+      assert_false m.has_b?
+      assert_false m.has_d?
+
+      m = OneofMessage.new
+      m.d = :A
+      assert_true m.has_d?
+      assert_equal :A, m.d
+      assert_true m.has_my_oneof?
+      assert_false m.has_a?
+      assert_false m.has_b?
+      assert_false m.has_c?
+    end
+
+    def test_defined_defaults
+      m = TestMessageDefaults.new
+      assert_equal 1, m.optional_int32
+      assert_equal 2, m.optional_int64
+      assert_equal 3, m.optional_uint32
+      assert_equal 4, m.optional_uint64
+      assert_equal true, m.optional_bool
+      assert_equal 6.0, m.optional_float
+      assert_equal 7.0, m.optional_double
+      assert_equal "Default Str", m.optional_string
+      assert_equal "\xCF\xA5s\xBD\xBA\xE6fubar".force_encoding("ASCII-8BIT"), m.optional_bytes
+      assert_equal :B2, m.optional_enum
+
+      assert_false m.has_optional_int32?
+      assert_false m.has_optional_int64?
+      assert_false m.has_optional_uint32?
+      assert_false m.has_optional_uint64?
+      assert_false m.has_optional_bool?
+      assert_false m.has_optional_float?
+      assert_false m.has_optional_double?
+      assert_false m.has_optional_string?
+      assert_false m.has_optional_bytes?
+      assert_false m.has_optional_enum?
+    end
+
+    def test_set_clear_defaults
+      m = TestMessageDefaults.new
+
+      m.optional_int32 = -42
+      assert_equal -42, m.optional_int32
+      assert_true m.has_optional_int32?
+      m.clear_optional_int32
+      assert_equal 1, m.optional_int32
+      assert_false m.has_optional_int32?
+
+      m.optional_string = "foo bar"
+      assert_equal "foo bar", m.optional_string
+      assert_true m.has_optional_string?
+      m.clear_optional_string
+      assert_equal "Default Str", m.optional_string
+      assert_false m.has_optional_string?
+
+      m.optional_msg = TestMessage2.new(:foo => 42)
+      assert_equal TestMessage2.new(:foo => 42), m.optional_msg
+      assert_true m.has_optional_msg?
+
+      m.clear_optional_msg
+      assert_equal nil, m.optional_msg
+      assert_false m.has_optional_msg?
+
+      m.optional_msg = TestMessage2.new(:foo => 42)
+      assert_equal TestMessage2.new(:foo => 42), m.optional_msg
+      assert_true TestMessageDefaults.descriptor.lookup('optional_msg').has?(m)
+
+      TestMessageDefaults.descriptor.lookup('optional_msg').clear(m)
+      assert_equal nil, m.optional_msg
+      assert_false TestMessageDefaults.descriptor.lookup('optional_msg').has?(m)
+
+      m = TestMessage.new
+      m.repeated_int32.push(1)
+      assert_equal [1], m.repeated_int32
+      m.clear_repeated_int32
+      assert_equal [], m.repeated_int32
+
+      m = OneofMessage.new
+      m.a = "foo"
+      assert_equal "foo", m.a
+      assert_true m.has_a?
+      m.clear_a
+      assert_false m.has_a?
+
+      m = OneofMessage.new
+      m.a = "foobar"
+      assert_true m.has_my_oneof?
+      m.clear_my_oneof
+      assert_false m.has_my_oneof?
+
+      m = OneofMessage.new
+      m.a = "bar"
+      assert_equal "bar", m.a
+      assert_true m.has_my_oneof?
+      OneofMessage.descriptor.lookup('a').clear(m)
+      assert_false m.has_my_oneof?
+    end
+
+    def test_initialization_map_errors
+      e = assert_raise ArgumentError do
+        TestMessage.new(:hello => "world")
+      end
+      assert_match(/hello/, e.message)
+
+      e = assert_raise ArgumentError do
+        TestMessage.new(:repeated_uint32 => "hello")
+      end
+      assert_equal e.message, "Expected array as initializer value for repeated field 'repeated_uint32'."
+    end
+
+
+    def test_to_h
+      m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'])
+      expected_result = {
+        :optional_bool=>true,
+        :optional_double=>-10.100001,
+        :optional_string=>"foo",
+        :repeated_string=>["bar1", "bar2"],
+      }
+      assert_equal expected_result, m.to_h
+
+      m = OneofMessage.new(:a => "foo")
+      expected_result = {:a => "foo"}
+      assert_equal expected_result, m.to_h
+    end
+
+    def test_map_keyword_disabled
+      pool = Google::Protobuf::DescriptorPool.new
+
+      e = assert_raise ArgumentError do
+        pool.build do
+          add_file 'test_file.proto', syntax: :proto2 do
+            add_message "MapMessage" do
+              map :map_string_int32, :string, :int32, 1
+              map :map_string_msg, :string, :message, 2, "TestMessage2"
+            end
+          end
+        end
+      end
+
+      assert_match(/Cannot add a native map/, e.message)
+    end
+
+    def test_respond_to
+      # This test fails with JRuby 1.7.23, likely because of an old JRuby bug.
+      return if RUBY_PLATFORM == "java"
+      msg = TestMessage.new
+      assert !msg.respond_to?(:bacon)
+    end
+
+    def test_file_descriptor
+      file_descriptor = TestMessage.descriptor.file_descriptor
+      assert_true nil != file_descriptor
+      assert_equal "tests/basic_test_proto2.proto", file_descriptor.name
+      assert_equal :proto2, file_descriptor.syntax
+
+      file_descriptor = TestEnum.descriptor.file_descriptor
+      assert_true nil != file_descriptor
+      assert_equal "tests/basic_test_proto2.proto", file_descriptor.name
+      assert_equal :proto2, file_descriptor.syntax
+    end
+  end
+end

ruby/tests/basic_test.proto

+syntax = "proto3";
+
+package basic_test;
+
+message Foo {
+  Bar bar = 1;
+  repeated Baz baz = 2;
+}
+
+message Bar {
+  string msg = 1;
+}
+
+message Baz {
+  string msg = 1;
+}
+
+message TestMessage {
+  int32 optional_int32 = 1;
+  int64 optional_int64 = 2;
+  uint32 optional_uint32 = 3;
+  uint64 optional_uint64 = 4;
+  bool optional_bool = 5;
+  float optional_float = 6;
+  double optional_double = 7;
+  string optional_string = 8;
+  bytes optional_bytes = 9;
+  TestMessage2 optional_msg = 10;
+  TestEnum optional_enum = 11;
+
+  repeated int32 repeated_int32 = 12;
+  repeated int64 repeated_int64 = 13;
+  repeated uint32 repeated_uint32 = 14;
+  repeated uint64 repeated_uint64 = 15;
+  repeated bool repeated_bool = 16;
+  repeated float repeated_float = 17;
+  repeated double repeated_double = 18;
+  repeated string repeated_string = 19;
+  repeated bytes repeated_bytes = 20;
+  repeated TestMessage2 repeated_msg = 21;
+  repeated TestEnum repeated_enum = 22;
+}
+
+message TestMessage2 {
+  int32 foo = 1;
+}
+
+enum TestEnum {
+  Default = 0;
+  A = 1;
+  B = 2;
+  C = 3;
+}
+
+message TestEmbeddedMessageParent {
+  TestEmbeddedMessageChild child_msg = 1;
+  int32 number = 2;
+
+  repeated TestEmbeddedMessageChild repeated_msg = 3;
+  repeated int32 repeated_number = 4;
+}
+
+message TestEmbeddedMessageChild {
+  TestMessage sub_child = 1;
+}
+
+message Recursive1 {
+  Recursive2 foo = 1;
+}
+
+message Recursive2 {
+  Recursive1 foo = 1;
+}
+
+message MapMessage {
+  map<string, int32> map_string_int32 = 1;
+  map<string, TestMessage2> map_string_msg = 2;
+}
+
+message MapMessageWireEquiv {
+  repeated MapMessageWireEquiv_entry1 map_string_int32 = 1;
+  repeated MapMessageWireEquiv_entry2 map_string_msg = 2;
+}
+
+message MapMessageWireEquiv_entry1 {
+  string key = 1;
+  int32 value = 2;
+}
+
+message MapMessageWireEquiv_entry2 {
+  string key = 1;
+  TestMessage2 value = 2;
+}
+
+message OneofMessage {
+  oneof my_oneof {
+    string a = 1;
+    int32 b = 2;
+    TestMessage2 c = 3;
+    TestEnum d = 4;
+  }
+}
+
+message Outer {
+  map<int32, Inner> items = 1;
+}
+
+message Inner {
+}
\ No newline at end of file

ruby/tests/basic_test_proto2.proto

+syntax = "proto2";
+
+package basic_test_proto2;
+
+message Foo {
+  optional Bar bar = 1;
+  repeated Baz baz = 2;
+}
+
+message Bar {
+  optional string msg = 1;
+}
+
+message Baz {
+  optional string msg = 1;
+}
+
+message TestMessage {
+  optional int32 optional_int32 = 1;
+  optional int64 optional_int64 = 2;
+  optional uint32 optional_uint32 = 3;
+  optional uint64 optional_uint64 = 4;
+  optional bool optional_bool = 5;
+  optional float optional_float = 6;
+  optional double optional_double = 7;
+  optional string optional_string = 8;
+  optional bytes optional_bytes = 9;
+  optional TestMessage2 optional_msg = 10;
+  optional TestEnum optional_enum = 11;
+
+  repeated int32 repeated_int32 = 12;
+  repeated int64 repeated_int64 = 13;
+  repeated uint32 repeated_uint32 = 14;
+  repeated uint64 repeated_uint64 = 15;
+  repeated bool repeated_bool = 16;
+  repeated float repeated_float = 17;
+  repeated double repeated_double = 18;
+  repeated string repeated_string = 19;
+  repeated bytes repeated_bytes = 20;
+  repeated TestMessage2 repeated_msg = 21;
+  repeated TestEnum repeated_enum = 22;
+}
+
+message TestMessage2 {
+  optional int32 foo = 1;
+}
+
+message TestMessageDefaults {
+  optional int32 optional_int32 = 1 [default = 1];
+  optional int64 optional_int64 = 2 [default = 2];
+  optional uint32 optional_uint32 = 3 [default = 3];
+  optional uint64 optional_uint64 = 4 [default = 4];
+  optional bool optional_bool = 5 [default = true];
+  optional float optional_float = 6 [default = 6];
+  optional double optional_double = 7 [default = 7];
+  optional string optional_string = 8 [default = "Default Str"];
+  optional bytes optional_bytes = 9 [default = "\xCF\xA5s\xBD\xBA\xE6fubar"];
+  optional TestMessage2 optional_msg = 10;
+  optional TestNonZeroEnum optional_enum = 11 [default = B2];
+}
+
+enum TestEnum {
+  Default = 0;
+  A = 1;
+  B = 2;
+  C = 3;
+}
+
+enum TestNonZeroEnum {
+  A2 = 1;
+  B2 = 2;
+  C2 = 3;
+}
+
+message TestEmbeddedMessageParent {
+  optional TestEmbeddedMessageChild child_msg = 1;
+  optional int32 number = 2;
+
+  repeated TestEmbeddedMessageChild repeated_msg = 3;
+  repeated int32 repeated_number = 4;
+}
+
+message TestEmbeddedMessageChild {
+  optional TestMessage sub_child = 1;
+}
+
+message Recursive1 {
+  optional Recursive2 foo = 1;
+}
+
+message Recursive2 {
+  optional Recursive1 foo = 1;
+}
+
+message MapMessageWireEquiv {
+  repeated MapMessageWireEquiv_entry1 map_string_int32 = 1;
+  repeated MapMessageWireEquiv_entry2 map_string_msg = 2;
+}
+
+message MapMessageWireEquiv_entry1 {
+  optional string key = 1;
+  optional int32 value = 2;
+}
+
+message MapMessageWireEquiv_entry2 {
+  optional string key = 1;
+  optional TestMessage2 value = 2;
+}
+
+message OneofMessage {
+  oneof my_oneof {
+    string a = 1;
+    int32 b = 2;
+    TestMessage2 c = 3;
+    TestEnum d = 4;
+  }
+}

ruby/tests/test_import_proto2.proto

+syntax = "proto2";
+
+package foo_bar.proto2;
+
+message TestImportedMessage {}

ruby/tests/test_ruby_package_proto2.proto

+syntax = "proto2";
+
+package foo_bar_proto2;
+
+option ruby_package = "A.B.Proto2";
+
+message TestRubyPackageMessage {}