Reserve unknown in Ruby (#3763)

* Reserve unknown in ruby

* Revert ruby tests. Wait for cpp impl for conformance test

* Add conformance test for preserving unknown

* Add unknown field conformance test to csharp failure list.

* Fix comments

* Fix comment

* Fix comments

* Fix typo

* Use stringsink_string directly

* Mark hd unused

* Remove unused encodeunknown_handlerfunc

.gitignore

@@ -176,3 +176,7 @@ js/testproto_libs2.js
 ruby/lib/
 ruby/tests/generated_code_pb.rb
 ruby/tests/test_import_pb.rb
+ruby/Gemfile.lock
+ruby/compatibility_tests/v3.0.0/protoc
+ruby/compatibility_tests/v3.0.0/tests/generated_code_pb.rb
+ruby/compatibility_tests/v3.0.0/tests/test_import_pb.rb

conformance/conformance_php.php

@@ -57,10 +57,10 @@ function doTest($request)
           return $response;
         }
       } elseif ($request->getMessageType() == "protobuf_test_messages.proto2.TestAllTypesProto2") {
-	$response->setSkipped("PHP doesn't support proto2");
-	return $response;
+        $response->setSkipped("PHP doesn't support proto2");
+        return $response;
       } else {
-	trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
+        trigger_error("Protobuf request doesn't have specific payload type", E_USER_ERROR);
       }
     } elseif ($request->getPayload() == "json_payload") {
       try {

conformance/conformance_test.cc

@@ -290,6 +290,28 @@ void ConformanceTestSuite::RunValidInputTest(
       TextFormat::ParseFromString(equivalent_text_format, reference_message))
           << "Failed to parse data for test case: " << test_name
           << ", data: " << equivalent_text_format;
+  const string equivalent_wire_format = reference_message->SerializeAsString();
+  RunValidBinaryInputTest(test_name, level, input, input_format,
+                          equivalent_wire_format, requested_output, isProto3);
+}
+
+void ConformanceTestSuite::RunValidBinaryInputTest(
+    const string& test_name, ConformanceLevel level, const string& input,
+    WireFormat input_format, const string& equivalent_wire_format,
+    WireFormat requested_output, bool isProto3) {
+  auto newTestMessage = [&isProto3]() {
+    Message* newMessage;
+    if (isProto3) {
+      newMessage = new TestAllTypesProto3;
+    } else {
+      newMessage = new TestAllTypesProto2;
+    }
+    return newMessage;
+  };
+  Message* reference_message = newTestMessage();
+  GOOGLE_CHECK(
+      reference_message->ParseFromString(equivalent_wire_format))
+          << "Failed to parse wire data for test case: " << test_name;
 
   ConformanceRequest request;
   ConformanceResponse response;
@@ -493,6 +515,19 @@ void ConformanceTestSuite::RunValidProtobufTest(
   }
 }
 
+void ConformanceTestSuite::RunValidBinaryProtobufTest(
+    const string& test_name, ConformanceLevel level,
+    const string& input_protobuf, bool isProto3) {
+  string rname = ".Proto3";
+  if (!isProto3) {
+    rname = ".Proto2";
+  }
+  RunValidBinaryInputTest(
+      ConformanceLevelToString(level) + rname + ".ProtobufInput." + test_name +
+      ".ProtobufOutput", level, input_protobuf, conformance::PROTOBUF,
+      input_protobuf, conformance::PROTOBUF, isProto3);
+}
+
 void ConformanceTestSuite::RunValidProtobufTestWithMessage(
     const string& test_name, ConformanceLevel level, const Message *input,
     const string& equivalent_text_format, bool isProto3) {
@@ -811,6 +846,14 @@ void ConformanceTestSuite::TestOneofMessage (MessageType &message,
       "OneofZeroEnum", RECOMMENDED, &message, "oneof_enum: FOO", isProto3);
 }
 
+template <class MessageType>
+void ConformanceTestSuite::TestUnknownMessage(MessageType& message,
+                                              bool isProto3) {
+  message.ParseFromString("\xA8\x1F\x01");
+  RunValidBinaryProtobufTest("UnknownVarint", REQUIRED,
+                             message.SerializeAsString(), isProto3);
+}
+
 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
                                     std::string* output) {
   runner_ = runner;
@@ -1847,6 +1890,14 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
       "StringFieldSingleQuoteBoth", RECOMMENDED,
       R"({'optionalString': 'Hello world!'})");
 
+  // Unknown fields.
+  {
+    TestAllTypesProto3 messageProto3;
+    TestAllTypesProto2 messageProto2;
+    TestUnknownMessage(messageProto3, true);
+    TestUnknownMessage(messageProto2, false);
+  }
+
   // Wrapper types.
   RunValidJsonTest(
       "OptionalBoolWrapper", REQUIRED,

conformance/conformance_test.h

@@ -167,6 +167,13 @@ class ConformanceTestSuite {
                          const string& equivalent_text_format,
                          conformance::WireFormat requested_output,
                          bool isProto3);
+  void RunValidBinaryInputTest(const string& test_name,
+                               ConformanceLevel level,
+                               const string& input,
+                               conformance::WireFormat input_format,
+                               const string& equivalent_wire_format,
+                               conformance::WireFormat requested_output,
+                               bool isProto3);
   void RunValidJsonTest(const string& test_name,
                         ConformanceLevel level,
                         const string& input_json,
@@ -180,6 +187,10 @@ class ConformanceTestSuite {
                             const string& input_protobuf,
                             const string& equivalent_text_format,
                             bool isProto3);
+  void RunValidBinaryProtobufTest(const string& test_name,
+                                  ConformanceLevel level,
+                                  const string& input_protobuf,
+                                  bool isProto3);
   void RunValidProtobufTestWithMessage(
       const string& test_name, ConformanceLevel level,
       const Message *input,
@@ -212,6 +223,9 @@ class ConformanceTestSuite {
   template <class MessageType>
   void TestOneofMessage (MessageType &message,
                          bool isProto3);
+  template <class MessageType>
+  void TestUnknownMessage (MessageType &message,
+                           bool isProto3);
   void TestValidDataForType(
       google::protobuf::FieldDescriptor::Type,
       std::vector<std::pair<std::string, std::string>> values);

conformance/failure_list_csharp.txt

 Recommended.Proto3.JsonInput.BytesFieldBase64Url.JsonOutput
 Recommended.Proto3.JsonInput.BytesFieldBase64Url.ProtobufOutput
+Required.Proto3.ProtobufInput.UnknownVarint.ProtobufOutput

conformance/failure_list_ruby.txt

@@ -59,8 +59,6 @@ Required.Proto3.JsonInput.FieldMask.ProtobufOutput
 Required.Proto3.JsonInput.FloatFieldInfinity.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNan.JsonOutput
 Required.Proto3.JsonInput.FloatFieldNegativeInfinity.JsonOutput
-Required.Proto3.JsonInput.FloatFieldTooLarge
-Required.Proto3.JsonInput.FloatFieldTooSmall
 Required.Proto3.JsonInput.OneofFieldDuplicate
 Required.Proto3.JsonInput.OptionalBoolWrapper.JsonOutput
 Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput

ruby/ext/google/protobuf_c/encode_decode.c

@@ -44,6 +44,56 @@ VALUE noleak_rb_str_cat(VALUE rb_str, const char *str, long len) {
   return rb_str;
 }
 
+// The code below also comes from upb's prototype Ruby binding, developed by
+// haberman@.
+
+/* stringsink *****************************************************************/
+
+static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) {
+  stringsink *sink = _sink;
+  sink->len = 0;
+  return sink;
+}
+
+static size_t stringsink_string(void *_sink, const void *hd, const char *ptr,
+                                size_t len, const upb_bufhandle *handle) {
+  stringsink *sink = _sink;
+  size_t new_size = sink->size;
+
+  UPB_UNUSED(hd);
+  UPB_UNUSED(handle);
+
+  while (sink->len + len > new_size) {
+    new_size *= 2;
+  }
+
+  if (new_size != sink->size) {
+    sink->ptr = realloc(sink->ptr, new_size);
+    sink->size = new_size;
+  }
+
+  memcpy(sink->ptr + sink->len, ptr, len);
+  sink->len += len;
+
+  return len;
+}
+
+void stringsink_init(stringsink *sink) {
+  upb_byteshandler_init(&sink->handler);
+  upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL);
+  upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL);
+
+  upb_bytessink_reset(&sink->sink, &sink->handler, sink);
+
+  sink->size = 32;
+  sink->ptr = malloc(sink->size);
+  sink->len = 0;
+}
+
+void stringsink_uninit(stringsink *sink) {
+  free(sink->ptr);
+}
+
 // -----------------------------------------------------------------------------
 // Parsing.
 // -----------------------------------------------------------------------------
@@ -613,6 +663,20 @@ static void add_handlers_for_oneof_field(upb_handlers *h,
   upb_handlerattr_uninit(&attr);
 }
 
+static bool unknown_field_handler(void* closure, const void* hd,
+                                  const char* buf, size_t size) {
+  UPB_UNUSED(hd);
+
+  MessageHeader* msg = (MessageHeader*)closure;
+  if (msg->unknown_fields == NULL) {
+    msg->unknown_fields = malloc(sizeof(stringsink));
+    stringsink_init(msg->unknown_fields);
+  }
+
+  stringsink_string(msg->unknown_fields, NULL, buf, size, NULL);
+
+  return true;
+}
 
 static void add_handlers_for_message(const void *closure, upb_handlers *h) {
   const upb_msgdef* msgdef = upb_handlers_msgdef(h);
@@ -634,6 +698,9 @@ static void add_handlers_for_message(const void *closure, upb_handlers *h) {
     desc->layout = create_layout(desc->msgdef);
   }
 
+  upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+  upb_handlers_setunknown(h, unknown_field_handler, &attr);
+
   for (upb_msg_field_begin(&i, desc->msgdef);
        !upb_msg_field_done(&i);
        upb_msg_field_next(&i)) {
@@ -831,65 +898,6 @@ VALUE Message_decode_json(VALUE klass, VALUE data) {
 // -----------------------------------------------------------------------------
 // Serializing.
 // -----------------------------------------------------------------------------
-//
-// The code below also comes from upb's prototype Ruby binding, developed by
-// haberman@.
-
-/* stringsink *****************************************************************/
-
-// This should probably be factored into a common upb component.
-
-typedef struct {
-  upb_byteshandler handler;
-  upb_bytessink sink;
-  char *ptr;
-  size_t len, size;
-} stringsink;
-
-static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) {
-  stringsink *sink = _sink;
-  sink->len = 0;
-  return sink;
-}
-
-static size_t stringsink_string(void *_sink, const void *hd, const char *ptr,
-                                size_t len, const upb_bufhandle *handle) {
-  stringsink *sink = _sink;
-  size_t new_size = sink->size;
-
-  UPB_UNUSED(hd);
-  UPB_UNUSED(handle);
-
-  while (sink->len + len > new_size) {
-    new_size *= 2;
-  }
-
-  if (new_size != sink->size) {
-    sink->ptr = realloc(sink->ptr, new_size);
-    sink->size = new_size;
-  }
-
-  memcpy(sink->ptr + sink->len, ptr, len);
-  sink->len += len;
-
-  return len;
-}
-
-void stringsink_init(stringsink *sink) {
-  upb_byteshandler_init(&sink->handler);
-  upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL);
-  upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL);
-
-  upb_bytessink_reset(&sink->sink, &sink->handler, sink);
-
-  sink->size = 32;
-  sink->ptr = malloc(sink->size);
-  sink->len = 0;
-}
-
-void stringsink_uninit(stringsink *sink) {
-  free(sink->ptr);
-}
 
 /* msgvisitor *****************************************************************/
 
@@ -1171,6 +1179,11 @@ static void putmsg(VALUE msg_rb, const Descriptor* desc,
     }
   }
 
+  stringsink* unknown = msg->unknown_fields;
+  if (unknown != NULL) {
+    upb_sink_putunknown(sink, unknown->ptr, unknown->len);
+  }
+
   upb_sink_endmsg(sink, &status);
 }
 

ruby/ext/google/protobuf_c/message.c

@@ -44,6 +44,11 @@ void Message_mark(void* _self) {
 }
 
 void Message_free(void* self) {
+  stringsink* unknown = ((MessageHeader *)self)->unknown_fields;
+  if (unknown != NULL) {
+    stringsink_uninit(unknown);
+    free(unknown);
+  }
   xfree(self);
 }
 
@@ -67,6 +72,8 @@ VALUE Message_alloc(VALUE klass) {
   msg->descriptor = desc;
   rb_ivar_set(ret, descriptor_instancevar_interned, descriptor);
 
+  msg->unknown_fields = NULL;
+
   layout_init(desc->layout, Message_data(msg));
 
   return ret;

ruby/ext/google/protobuf_c/protobuf.h

@@ -475,8 +475,20 @@ VALUE layout_inspect(MessageLayout* layout, void* storage);
 // Message class creation.
 // -----------------------------------------------------------------------------
 
+// This should probably be factored into a common upb component.
+
+typedef struct {
+  upb_byteshandler handler;
+  upb_bytessink sink;
+  char *ptr;
+  size_t len, size;
+} stringsink;
+
+void stringsink_uninit(stringsink *sink);
+
 struct MessageHeader {
-  Descriptor* descriptor;  // kept alive by self.class.descriptor reference.
+  Descriptor* descriptor;      // kept alive by self.class.descriptor reference.
+  stringsink* unknown_fields;  // store unknown fields in decoding.
   // Data comes after this.
 };
 

ruby/ext/google/protobuf_c/upb.h

 // Amalgamated source file
 /*
-** Defs are upb's internal representation of the constructs that can appear
-** in a .proto file:
-**
-** - upb::MessageDef (upb_msgdef): describes a "message" construct.
-** - upb::FieldDef (upb_fielddef): describes a message field.
-** - upb::FileDef (upb_filedef): describes a .proto file and its defs.
-** - upb::EnumDef (upb_enumdef): describes an enum.
-** - upb::OneofDef (upb_oneofdef): describes a oneof.
-** - upb::Def (upb_def): base class of all the others.
-**
-** TODO: definitions of services.
-**
-** Like upb_refcounted objects, defs are mutable only until frozen, and are
-** only thread-safe once frozen.
-**
-** This is a mixed C/C++ interface that offers a full API to both languages.
-** See the top-level README for more information.
-*/
-
-#ifndef UPB_DEF_H_
-#define UPB_DEF_H_
-
-/*
-** upb::RefCounted (upb_refcounted)
-**
-** A refcounting scheme that supports circular refs.  It accomplishes this by
-** partitioning the set of objects into groups such that no cycle spans groups;
-** we can then reference-count the group as a whole and ignore refs within the
-** group.  When objects are mutable, these groups are computed very
-** conservatively; we group any objects that have ever had a link between them.
-** When objects are frozen, we compute strongly-connected components which
-** allows us to be precise and only group objects that are actually cyclic.
-**
-** This is a mixed C/C++ interface that offers a full API to both languages.
-** See the top-level README for more information.
-*/
-
-#ifndef UPB_REFCOUNTED_H_
-#define UPB_REFCOUNTED_H_
-
-/*
-** upb_table
-**
-** This header is INTERNAL-ONLY!  Its interfaces are not public or stable!
-** This file defines very fast int->upb_value (inttable) and string->upb_value
-** (strtable) hash tables.
-**
-** The table uses chained scatter with Brent's variation (inspired by the Lua
-** implementation of hash tables).  The hash function for strings is Austin
-** Appleby's "MurmurHash."
-**
-** The inttable uses uintptr_t as its key, which guarantees it can be used to
-** store pointers or integers of at least 32 bits (upb isn't really useful on
-** systems where sizeof(void*) < 4).
-**
-** The table must be homogenous (all values of the same type).  In debug
-** mode, we check this on insert and lookup.
-*/
-
-#ifndef UPB_TABLE_H_
-#define UPB_TABLE_H_
-
-#include <stdint.h>
-#include <string.h>
-/*
 ** This file contains shared definitions that are widely used across upb.
 **
 ** This is a mixed C/C++ interface that offers a full API to both languages.
@@ -100,6 +35,9 @@ template <int N> class InlinedEnvironment;
 #define UPB_INLINE static
 #endif
 
+/* Hints to the compiler about likely/unlikely branches. */
+#define UPB_LIKELY(x) __builtin_expect((x),1)
+
 /* Define UPB_BIG_ENDIAN manually if you're on big endian and your compiler
  * doesn't provide these preprocessor symbols. */
 #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
@@ -361,6 +299,16 @@ class PointerBase2 : public PointerBase<T, Base> {
 
 #endif
 
+/* A list of types as they are encoded on-the-wire. */
+typedef enum {
+  UPB_WIRE_TYPE_VARINT      = 0,
+  UPB_WIRE_TYPE_64BIT       = 1,
+  UPB_WIRE_TYPE_DELIMITED   = 2,
+  UPB_WIRE_TYPE_START_GROUP = 3,
+  UPB_WIRE_TYPE_END_GROUP   = 4,
+  UPB_WIRE_TYPE_32BIT       = 5
+} upb_wiretype_t;
+
 
 /* upb::ErrorSpace ************************************************************/
 
@@ -689,7 +637,7 @@ void upb_env_uninit(upb_env *e);
 
 void upb_env_initonly(upb_env *e);
 
-upb_arena *upb_env_arena(upb_env *e);
+UPB_INLINE upb_arena *upb_env_arena(upb_env *e) { return (upb_arena*)e; }
 bool upb_env_ok(const upb_env *e);
 void upb_env_seterrorfunc(upb_env *e, upb_error_func *func, void *ud);
 
@@ -789,6 +737,106 @@ template <int N> class upb::InlinedEnvironment : public upb::Environment {
 
 
 #endif  /* UPB_H_ */
+/*
+** upb_decode: parsing into a upb_msg using a upb_msglayout.
+*/
+
+#ifndef UPB_DECODE_H_
+#define UPB_DECODE_H_
+
+/*
+** upb::Message is a representation for protobuf messages.
+**
+** However it differs from other common representations like
+** google::protobuf::Message in one key way: it does not prescribe any
+** ownership between messages and submessages, and it relies on the
+** client to delete each message/submessage/array/map at the appropriate
+** time.
+**
+** A client can access a upb::Message without knowing anything about
+** ownership semantics, but to create or mutate a message a user needs
+** to implement the memory management themselves.
+**
+** Currently all messages, arrays, and maps store a upb_alloc* internally.
+** Mutating operations use this when they require dynamically-allocated
+** memory.  We could potentially eliminate this size overhead later by
+** letting the user flip a bit on the factory that prevents this from
+** being stored.  The user would then need to use separate functions where
+** the upb_alloc* is passed explicitly.  However for handlers to populate
+** such structures, they would need a place to store this upb_alloc* during
+** parsing; upb_handlers don't currently have a good way to accommodate this.
+**
+** TODO: UTF-8 checking?
+**/
+
+#ifndef UPB_MSG_H_
+#define UPB_MSG_H_
+
+/*
+** Defs are upb's internal representation of the constructs that can appear
+** in a .proto file:
+**
+** - upb::MessageDef (upb_msgdef): describes a "message" construct.
+** - upb::FieldDef (upb_fielddef): describes a message field.
+** - upb::FileDef (upb_filedef): describes a .proto file and its defs.
+** - upb::EnumDef (upb_enumdef): describes an enum.
+** - upb::OneofDef (upb_oneofdef): describes a oneof.
+** - upb::Def (upb_def): base class of all the others.
+**
+** TODO: definitions of services.
+**
+** Like upb_refcounted objects, defs are mutable only until frozen, and are
+** only thread-safe once frozen.
+**
+** This is a mixed C/C++ interface that offers a full API to both languages.
+** See the top-level README for more information.
+*/
+
+#ifndef UPB_DEF_H_
+#define UPB_DEF_H_
+
+/*
+** upb::RefCounted (upb_refcounted)
+**
+** A refcounting scheme that supports circular refs.  It accomplishes this by
+** partitioning the set of objects into groups such that no cycle spans groups;
+** we can then reference-count the group as a whole and ignore refs within the
+** group.  When objects are mutable, these groups are computed very
+** conservatively; we group any objects that have ever had a link between them.
+** When objects are frozen, we compute strongly-connected components which
+** allows us to be precise and only group objects that are actually cyclic.
+**
+** This is a mixed C/C++ interface that offers a full API to both languages.
+** See the top-level README for more information.
+*/
+
+#ifndef UPB_REFCOUNTED_H_
+#define UPB_REFCOUNTED_H_
+
+/*
+** upb_table
+**
+** This header is INTERNAL-ONLY!  Its interfaces are not public or stable!
+** This file defines very fast int->upb_value (inttable) and string->upb_value
+** (strtable) hash tables.
+**
+** The table uses chained scatter with Brent's variation (inspired by the Lua
+** implementation of hash tables).  The hash function for strings is Austin
+** Appleby's "MurmurHash."
+**
+** The inttable uses uintptr_t as its key, which guarantees it can be used to
+** store pointers or integers of at least 32 bits (upb isn't really useful on
+** systems where sizeof(void*) < 4).
+**
+** The table must be homogenous (all values of the same type).  In debug
+** mode, we check this on insert and lookup.
+*/
+
+#ifndef UPB_TABLE_H_
+#define UPB_TABLE_H_
+
+#include <stdint.h>
+#include <string.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -3851,265 +3899,75 @@ inline bool FileDef::AddDependency(const FileDef* file) {
 
 #endif /* UPB_DEF_H_ */
 /*
-** This file contains definitions of structs that should be considered private
-** and NOT stable across versions of upb.
+** upb::Handlers (upb_handlers)
 **
-** The only reason they are declared here and not in .c files is to allow upb
-** and the application (if desired) to embed statically-initialized instances
-** of structures like defs.
+** A upb_handlers is like a virtual table for a upb_msgdef.  Each field of the
+** message can have associated functions that will be called when we are
+** parsing or visiting a stream of data.  This is similar to how handlers work
+** in SAX (the Simple API for XML).
 **
-** If you include this file, all guarantees of ABI compatibility go out the
-** window!  Any code that includes this file needs to recompile against the
-** exact same version of upb that they are linking against.
+** The handlers have no idea where the data is coming from, so a single set of
+** handlers could be used with two completely different data sources (for
+** example, a parser and a visitor over in-memory objects).  This decoupling is
+** the most important feature of upb, because it allows parsers and serializers
+** to be highly reusable.
 **
-** You also need to recompile if you change the value of the UPB_DEBUG_REFS
-** flag.
+** This is a mixed C/C++ interface that offers a full API to both languages.
+** See the top-level README for more information.
 */
 
+#ifndef UPB_HANDLERS_H
+#define UPB_HANDLERS_H
 
-#ifndef UPB_STATICINIT_H_
-#define UPB_STATICINIT_H_
 
 #ifdef __cplusplus
-/* Because of how we do our typedefs, this header can't be included from C++. */
-#error This file cannot be included from C++
+namespace upb {
+class BufferHandle;
+class BytesHandler;
+class HandlerAttributes;
+class Handlers;
+template <class T> class Handler;
+template <class T> struct CanonicalType;
+}  /* namespace upb */
 #endif
 
-/* upb_refcounted *************************************************************/
-
+UPB_DECLARE_TYPE(upb::BufferHandle, upb_bufhandle)
+UPB_DECLARE_TYPE(upb::BytesHandler, upb_byteshandler)
+UPB_DECLARE_TYPE(upb::HandlerAttributes, upb_handlerattr)
+UPB_DECLARE_DERIVED_TYPE(upb::Handlers, upb::RefCounted,
+                         upb_handlers, upb_refcounted)
 
-/* upb_def ********************************************************************/
+/* The maximum depth that the handler graph can have.  This is a resource limit
+ * for the C stack since we sometimes need to recursively traverse the graph.
+ * Cycles are ok; the traversal will stop when it detects a cycle, but we must
+ * hit the cycle before the maximum depth is reached.
+ *
+ * If having a single static limit is too inflexible, we can add another variant
+ * of Handlers::Freeze that allows specifying this as a parameter. */
+#define UPB_MAX_HANDLER_DEPTH 64
 
-struct upb_def {
-  upb_refcounted base;
+/* All the different types of handlers that can be registered.
+ * Only needed for the advanced functions in upb::Handlers. */
+typedef enum {
+  UPB_HANDLER_INT32,
+  UPB_HANDLER_INT64,
+  UPB_HANDLER_UINT32,
+  UPB_HANDLER_UINT64,
+  UPB_HANDLER_FLOAT,
+  UPB_HANDLER_DOUBLE,
+  UPB_HANDLER_BOOL,
+  UPB_HANDLER_STARTSTR,
+  UPB_HANDLER_STRING,
+  UPB_HANDLER_ENDSTR,
+  UPB_HANDLER_STARTSUBMSG,
+  UPB_HANDLER_ENDSUBMSG,
+  UPB_HANDLER_STARTSEQ,
+  UPB_HANDLER_ENDSEQ
+} upb_handlertype_t;
 
-  const char *fullname;
-  const upb_filedef* file;
-  char type;  /* A upb_deftype_t (char to save space) */
+#define UPB_HANDLER_MAX (UPB_HANDLER_ENDSEQ+1)
 
-  /* Used as a flag during the def's mutable stage.  Must be false unless
-   * it is currently being used by a function on the stack.  This allows
-   * us to easily determine which defs were passed into the function's
-   * current invocation. */
-  bool came_from_user;
-};
-
-#define UPB_DEF_INIT(name, type, vtbl, refs, ref2s) \
-    { UPB_REFCOUNT_INIT(vtbl, refs, ref2s), name, NULL, type, false }
-
-
-/* upb_fielddef ***************************************************************/
-
-struct upb_fielddef {
-  upb_def base;
-
-  union {
-    int64_t sint;
-    uint64_t uint;
-    double dbl;
-    float flt;
-    void *bytes;
-  } defaultval;
-  union {
-    const upb_msgdef *def;  /* If !msg_is_symbolic. */
-    char *name;             /* If msg_is_symbolic. */
-  } msg;
-  union {
-    const upb_def *def;  /* If !subdef_is_symbolic. */
-    char *name;          /* If subdef_is_symbolic. */
-  } sub;  /* The msgdef or enumdef for this field, if upb_hassubdef(f). */
-  bool subdef_is_symbolic;
-  bool msg_is_symbolic;
-  const upb_oneofdef *oneof;
-  bool default_is_string;
-  bool type_is_set_;     /* False until type is explicitly set. */
-  bool is_extension_;
-  bool lazy_;
-  bool packed_;
-  upb_intfmt_t intfmt;
-  bool tagdelim;
-  upb_fieldtype_t type_;
-  upb_label_t label_;
-  uint32_t number_;
-  uint32_t selector_base;  /* Used to index into a upb::Handlers table. */
-  uint32_t index_;
-};
-
-extern const struct upb_refcounted_vtbl upb_fielddef_vtbl;
-
-#define UPB_FIELDDEF_INIT(label, type, intfmt, tagdelim, is_extension, lazy,   \
-                          packed, name, num, msgdef, subdef, selector_base,    \
-                          index, defaultval, refs, ref2s)                      \
-  {                                                                            \
-    UPB_DEF_INIT(name, UPB_DEF_FIELD, &upb_fielddef_vtbl, refs, ref2s),        \
-        defaultval, {msgdef}, {subdef}, NULL, false, false,                    \
-        type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES, true, is_extension, \
-        lazy, packed, intfmt, tagdelim, type, label, num, selector_base, index \
-  }
-
-
-/* upb_msgdef *****************************************************************/
-
-struct upb_msgdef {
-  upb_def base;
-
-  size_t selector_count;
-  uint32_t submsg_field_count;
-
-  /* Tables for looking up fields by number and name. */
-  upb_inttable itof;  /* int to field */
-  upb_strtable ntof;  /* name to field/oneof */
-
-  /* Is this a map-entry message? */
-  bool map_entry;
-
-  /* Whether this message has proto2 or proto3 semantics. */
-  upb_syntax_t syntax;
-
-  /* TODO(haberman): proper extension ranges (there can be multiple). */
-};
-
-extern const struct upb_refcounted_vtbl upb_msgdef_vtbl;
-
-/* TODO: also support static initialization of the oneofs table. This will be
- * needed if we compile in descriptors that contain oneofs. */
-#define UPB_MSGDEF_INIT(name, selector_count, submsg_field_count, itof, ntof, \
-                        map_entry, syntax, refs, ref2s)                       \
-  {                                                                           \
-    UPB_DEF_INIT(name, UPB_DEF_MSG, &upb_fielddef_vtbl, refs, ref2s),         \
-        selector_count, submsg_field_count, itof, ntof, map_entry, syntax     \
-  }
-
-
-/* upb_enumdef ****************************************************************/
-
-struct upb_enumdef {
-  upb_def base;
-
-  upb_strtable ntoi;
-  upb_inttable iton;
-  int32_t defaultval;
-};
-
-extern const struct upb_refcounted_vtbl upb_enumdef_vtbl;
-
-#define UPB_ENUMDEF_INIT(name, ntoi, iton, defaultval, refs, ref2s) \
-  { UPB_DEF_INIT(name, UPB_DEF_ENUM, &upb_enumdef_vtbl, refs, ref2s), ntoi,    \
-    iton, defaultval }
-
-
-/* upb_oneofdef ***************************************************************/
-
-struct upb_oneofdef {
-  upb_refcounted base;
-
-  uint32_t index;  /* Index within oneofs. */
-  const char *name;
-  upb_strtable ntof;
-  upb_inttable itof;
-  const upb_msgdef *parent;
-};
-
-extern const struct upb_refcounted_vtbl upb_oneofdef_vtbl;
-
-#define UPB_ONEOFDEF_INIT(name, ntof, itof, refs, ref2s) \
-  { UPB_REFCOUNT_INIT(&upb_oneofdef_vtbl, refs, ref2s), 0, name, ntof, itof }
-
-
-/* upb_symtab *****************************************************************/
-
-struct upb_symtab {
-  upb_refcounted base;
-
-  upb_strtable symtab;
-};
-
-struct upb_filedef {
-  upb_refcounted base;
-
-  const char *name;
-  const char *package;
-  const char *phpprefix;
-  const char *phpnamespace;
-  upb_syntax_t syntax;
-
-  upb_inttable defs;
-  upb_inttable deps;
-};
-
-extern const struct upb_refcounted_vtbl upb_filedef_vtbl;
-
-#endif  /* UPB_STATICINIT_H_ */
-/*
-** upb::Handlers (upb_handlers)
-**
-** A upb_handlers is like a virtual table for a upb_msgdef.  Each field of the
-** message can have associated functions that will be called when we are
-** parsing or visiting a stream of data.  This is similar to how handlers work
-** in SAX (the Simple API for XML).
-**
-** The handlers have no idea where the data is coming from, so a single set of
-** handlers could be used with two completely different data sources (for
-** example, a parser and a visitor over in-memory objects).  This decoupling is
-** the most important feature of upb, because it allows parsers and serializers
-** to be highly reusable.
-**
-** This is a mixed C/C++ interface that offers a full API to both languages.
-** See the top-level README for more information.
-*/
-
-#ifndef UPB_HANDLERS_H
-#define UPB_HANDLERS_H
-
-
-#ifdef __cplusplus
-namespace upb {
-class BufferHandle;
-class BytesHandler;
-class HandlerAttributes;
-class Handlers;
-template <class T> class Handler;
-template <class T> struct CanonicalType;
-}  /* namespace upb */
-#endif
-
-UPB_DECLARE_TYPE(upb::BufferHandle, upb_bufhandle)
-UPB_DECLARE_TYPE(upb::BytesHandler, upb_byteshandler)
-UPB_DECLARE_TYPE(upb::HandlerAttributes, upb_handlerattr)
-UPB_DECLARE_DERIVED_TYPE(upb::Handlers, upb::RefCounted,
-                         upb_handlers, upb_refcounted)
-
-/* The maximum depth that the handler graph can have.  This is a resource limit
- * for the C stack since we sometimes need to recursively traverse the graph.
- * Cycles are ok; the traversal will stop when it detects a cycle, but we must
- * hit the cycle before the maximum depth is reached.
- *
- * If having a single static limit is too inflexible, we can add another variant
- * of Handlers::Freeze that allows specifying this as a parameter. */
-#define UPB_MAX_HANDLER_DEPTH 64
-
-/* All the different types of handlers that can be registered.
- * Only needed for the advanced functions in upb::Handlers. */
-typedef enum {
-  UPB_HANDLER_INT32,
-  UPB_HANDLER_INT64,
-  UPB_HANDLER_UINT32,
-  UPB_HANDLER_UINT64,
-  UPB_HANDLER_FLOAT,
-  UPB_HANDLER_DOUBLE,
-  UPB_HANDLER_BOOL,
-  UPB_HANDLER_STARTSTR,
-  UPB_HANDLER_STRING,
-  UPB_HANDLER_ENDSTR,
-  UPB_HANDLER_STARTSUBMSG,
-  UPB_HANDLER_ENDSUBMSG,
-  UPB_HANDLER_STARTSEQ,
-  UPB_HANDLER_ENDSEQ
-} upb_handlertype_t;
-
-#define UPB_HANDLER_MAX (UPB_HANDLER_ENDSEQ+1)
-
-#define UPB_BREAK NULL
+#define UPB_BREAK NULL
 
 /* A convenient definition for when no closure is needed. */
 extern char _upb_noclosure;
@@ -4144,7 +4002,8 @@ UPB_END_EXTERN_C
 /* Static selectors for upb::Handlers. */
 #define UPB_STARTMSG_SELECTOR 0
 #define UPB_ENDMSG_SELECTOR 1
-#define UPB_STATIC_SELECTOR_COUNT 2
+#define UPB_UNKNOWN_SELECTOR 2
+#define UPB_STATIC_SELECTOR_COUNT 3
 
 /* Static selectors for upb::BytesHandler. */
 #define UPB_STARTSTR_SELECTOR 0
@@ -4673,6 +4532,8 @@ UPB_BEGIN_EXTERN_C
 /* Native C API. */
 
 /* Handler function typedefs. */
+typedef bool upb_unknown_handlerfunc(void *c, const void *hd, const char *buf,
+                                     size_t n);
 typedef bool upb_startmsg_handlerfunc(void *c, const void*);
 typedef bool upb_endmsg_handlerfunc(void *c, const void *, upb_status *status);
 typedef void* upb_startfield_handlerfunc(void *c, const void *hd);
@@ -4726,6 +4587,8 @@ const upb_status *upb_handlers_status(upb_handlers *h);
 void upb_handlers_clearerr(upb_handlers *h);
 const upb_msgdef *upb_handlers_msgdef(const upb_handlers *h);
 bool upb_handlers_addcleanup(upb_handlers *h, void *p, upb_handlerfree *hfree);
+bool upb_handlers_setunknown(upb_handlers *h, upb_unknown_handlerfunc *func,
+                             upb_handlerattr *attr);
 
 bool upb_handlers_setstartmsg(upb_handlers *h, upb_startmsg_handlerfunc *func,
                               upb_handlerattr *attr);
@@ -6295,6 +6158,18 @@ UPB_INLINE size_t upb_sink_putstring(upb_sink *s, upb_selector_t sel,
   return handler(s->closure, hd, buf, n, handle);
 }
 
+UPB_INLINE bool upb_sink_putunknown(upb_sink *s, const char *buf, size_t n) {
+  typedef upb_unknown_handlerfunc func;
+  func *handler;
+  const void *hd;
+  if (!s->handlers) return true;
+  handler = (func *)upb_handlers_gethandler(s->handlers, UPB_UNKNOWN_SELECTOR);
+
+  if (!handler) return n;
+  hd = upb_handlers_gethandlerdata(s->handlers, UPB_UNKNOWN_SELECTOR);
+  return handler(s->closure, hd, buf, n);
+}
+
 UPB_INLINE bool upb_sink_startmsg(upb_sink *s) {
   typedef upb_startmsg_handlerfunc func;
   func *startmsg;
@@ -6499,34 +6374,6 @@ inline bool BufferSource::PutBuffer(const char *buf, size_t len,
 #endif
 
 #endif
-/*
-** upb::Message is a representation for protobuf messages.
-**
-** However it differs from other common representations like
-** google::protobuf::Message in one key way: it does not prescribe any
-** ownership between messages and submessages, and it relies on the
-** client to delete each message/submessage/array/map at the appropriate
-** time.
-**
-** A client can access a upb::Message without knowing anything about
-** ownership semantics, but to create or mutate a message a user needs
-** to implement the memory management themselves.
-**
-** Currently all messages, arrays, and maps store a upb_alloc* internally.
-** Mutating operations use this when they require dynamically-allocated
-** memory.  We could potentially eliminate this size overhead later by
-** letting the user flip a bit on the factory that prevents this from
-** being stored.  The user would then need to use separate functions where
-** the upb_alloc* is passed explicitly.  However for handlers to populate
-** such structures, they would need a place to store this upb_alloc* during
-** parsing; upb_handlers don't currently have a good way to accommodate this.
-**
-** TODO: UTF-8 checking?
-**/
-
-#ifndef UPB_MSG_H_
-#define UPB_MSG_H_
-
 
 #ifdef __cplusplus
 
@@ -6563,21 +6410,6 @@ typedef void upb_msg;
  * instances of this from a upb_msgfactory, and the factory always owns the
  * msglayout. */
 
-/* Gets the factory for this layout */
-upb_msgfactory *upb_msglayout_factory(const upb_msglayout *l);
-
-/* Get the msglayout for a submessage.  This requires that this field is a
- * submessage, ie. upb_fielddef_issubmsg(upb_msglayout_msgdef(l)) == true.
- *
- * Since map entry messages don't have layouts, if upb_fielddef_ismap(f) == true
- * then this function will return the layout for the map's value.  It requires
- * that the value type of the map field is a submessage. */
-const upb_msglayout *upb_msglayout_sublayout(const upb_msglayout *l,
-                                             const upb_fielddef *f);
-
-/* Returns the msgdef for this msglayout. */
-const upb_msgdef *upb_msglayout_msgdef(const upb_msglayout *l);
-
 
 /** upb_visitor ***************************************************************/
 
@@ -6623,6 +6455,23 @@ const upb_visitorplan *upb_msgfactory_getvisitorplan(upb_msgfactory *f,
                                                      const upb_handlers *h);
 
 
+/** upb_stringview ************************************************************/
+
+typedef struct {
+  const char *data;
+  size_t size;
+} upb_stringview;
+
+UPB_INLINE upb_stringview upb_stringview_make(const char *data, size_t size) {
+  upb_stringview ret;
+  ret.data = data;
+  ret.size = size;
+  return ret;
+}
+
+#define UPB_STRINGVIEW_INIT(ptr, len) {ptr, len}
+
+
 /** upb_msgval ****************************************************************/
 
 /* A union representing all possible protobuf values.  Used for generic get/set
@@ -6640,10 +6489,7 @@ typedef union {
   const upb_msg* msg;
   const upb_array* arr;
   const void* ptr;
-  struct {
-    const char *ptr;
-    size_t len;
-  } str;
+  upb_stringview str;
 } upb_msgval;
 
 #define ACCESSORS(name, membername, ctype) \
@@ -6670,22 +6516,12 @@ ACCESSORS(map,    map, const upb_map*)
 ACCESSORS(msg,    msg, const upb_msg*)
 ACCESSORS(ptr,    ptr, const void*)
 ACCESSORS(arr,    arr, const upb_array*)
+ACCESSORS(str,    str, upb_stringview)
 
 #undef ACCESSORS
 
-UPB_INLINE upb_msgval upb_msgval_str(const char *ptr, size_t len) {
-  upb_msgval ret;
-  ret.str.ptr = ptr;
-  ret.str.len = len;
-  return ret;
-}
-
-UPB_INLINE const char* upb_msgval_getstr(upb_msgval val) {
-  return val.str.ptr;
-}
-
-UPB_INLINE size_t upb_msgval_getstrlen(upb_msgval val) {
-  return val.str.len;
+UPB_INLINE upb_msgval upb_msgval_makestr(const char *data, size_t size) {
+  return upb_msgval_str(upb_stringview_make(data, size));
 }
 
 
@@ -6710,19 +6546,29 @@ size_t upb_msg_sizeof(const upb_msglayout *l);
  * upb_msg_uninit() must be called to release internally-allocated memory
  * unless the allocator is an arena that does not require freeing.
  *
+ * Please note that upb_msg_init() may return a value that is different than
+ * |msg|, so you must assign the return value and not cast your memory block
+ * to upb_msg* directly!
+ *
  * Please note that upb_msg_uninit() does *not* free any submessages, maps,
  * or arrays referred to by this message's fields.  You must free them manually
- * yourself. */
-void upb_msg_init(upb_msg *msg, const upb_msglayout *l, upb_alloc *a);
-void upb_msg_uninit(upb_msg *msg, const upb_msglayout *l);
+ * yourself.
+ *
+ * upb_msg_uninit returns the original memory block, which may be useful if
+ * you dynamically allocated it (though upb_msg_new() would normally be more
+ * appropriate in this case). */
+upb_msg *upb_msg_init(void *msg, const upb_msglayout *l, upb_alloc *a);
+void *upb_msg_uninit(upb_msg *msg, const upb_msglayout *l);
 
 /* Like upb_msg_init() / upb_msg_uninit(), except the message's memory is
  * allocated / freed from the given upb_alloc. */
 upb_msg *upb_msg_new(const upb_msglayout *l, upb_alloc *a);
 void upb_msg_free(upb_msg *msg, const upb_msglayout *l);
 
-/* Returns the upb_alloc for the given message. */
-upb_alloc *upb_msg_alloc(const upb_msg *msg, const upb_msglayout *l);
+/* Returns the upb_alloc for the given message.
+ * TODO(haberman): get rid of this?  Not sure we want to be storing this
+ * for every message. */
+upb_alloc *upb_msg_alloc(const upb_msg *msg);
 
 /* Packs the tree of messages rooted at "msg" into a single hunk of memory,
  * allocated from the given allocator. */
@@ -6742,25 +6588,14 @@ void *upb_msg_pack(const upb_msg *msg, const upb_msglayout *l,
  * arenas).
  */
 upb_msgval upb_msg_get(const upb_msg *msg,
-                       const upb_fielddef *f,
+                       int field_index,
                        const upb_msglayout *l);
 
 /* May only be called for fields where upb_fielddef_haspresence(f) == true. */
 bool upb_msg_has(const upb_msg *msg,
-                 const upb_fielddef *f,
+                 int field_index,
                  const upb_msglayout *l);
 
-/* Returns NULL if no field in the oneof is set. */
-const upb_fielddef *upb_msg_getoneofcase(const upb_msg *msg,
-                                         const upb_oneofdef *o,
-                                         const upb_msglayout *l);
-
-/* Returns true if any field in the oneof is set. */
-bool upb_msg_hasoneof(const upb_msg *msg,
-                      const upb_oneofdef *o,
-                      const upb_msglayout *l);
-
-
 /* Mutable message API.  May only be called by the owner of the message who
  * knows its ownership scheme and how to keep it consistent. */
 
@@ -6768,8 +6603,8 @@ bool upb_msg_hasoneof(const upb_msg *msg,
  * management: if you overwrite a pointer to a msg/array/map/string without
  * cleaning it up (or using an arena) it will leak.
  */
-bool upb_msg_set(upb_msg *msg,
-                 const upb_fielddef *f,
+void upb_msg_set(upb_msg *msg,
+                 int field_index,
                  upb_msgval val,
                  const upb_msglayout *l);
 
@@ -6780,12 +6615,7 @@ bool upb_msg_set(upb_msg *msg,
  * arrays/maps/strings/msgs that this field may have pointed to.
  */
 bool upb_msg_clearfield(upb_msg *msg,
-                        const upb_fielddef *f,
-                        const upb_msglayout *l);
-
-/* Clears all fields in the oneof such that none of them are set. */
-bool upb_msg_clearoneof(upb_msg *msg,
-                        const upb_oneofdef *o,
+                        int field_index,
                         const upb_msglayout *l);
 
 /* TODO(haberman): copyfrom()/mergefrom()? */
@@ -6898,9 +6728,288 @@ bool upb_msg_getscalarhandlerdata(const upb_handlers *h,
                                   size_t *offset,
                                   int32_t *hasbit);
 
+
+/** Interfaces for generated code *********************************************/
+
+#define UPB_NOT_IN_ONEOF UINT16_MAX
+#define UPB_NO_HASBIT UINT16_MAX
+#define UPB_NO_SUBMSG UINT16_MAX
+
+typedef struct {
+  uint32_t number;
+  uint32_t offset;  /* If in a oneof, offset of default in default_msg below. */
+  uint16_t hasbit;        /* UPB_NO_HASBIT if no hasbit. */
+  uint16_t oneof_index;   /* UPB_NOT_IN_ONEOF if not in a oneof. */
+  uint16_t submsg_index;  /* UPB_NO_SUBMSG if no submsg. */
+  uint8_t type;
+  uint8_t label;
+} upb_msglayout_fieldinit_v1;
+
+typedef struct {
+  uint32_t data_offset;
+  uint32_t case_offset;
+} upb_msglayout_oneofinit_v1;
+
+typedef struct upb_msglayout_msginit_v1 {
+  const struct upb_msglayout_msginit_v1 *const* submsgs;
+  const upb_msglayout_fieldinit_v1 *fields;
+  const upb_msglayout_oneofinit_v1 *oneofs;
+  void *default_msg;
+  /* Must be aligned to sizeof(void*).  Doesn't include internal members like
+   * unknown fields, extension dict, pointer to msglayout, etc. */
+  uint32_t size;
+  uint16_t field_count;
+  uint16_t oneof_count;
+  bool extendable;
+  bool is_proto2;
+} upb_msglayout_msginit_v1;
+
+#define UPB_ALIGN_UP_TO(val, align) ((val + (align - 1)) & -align)
+#define UPB_ALIGNED_SIZEOF(type) UPB_ALIGN_UP_TO(sizeof(type), sizeof(void*))
+
+/* Initialize/uninitialize a msglayout from a msginit.  If upb uses v1
+ * internally, this will not allocate any memory.  Should only be used by
+ * generated code. */
+upb_msglayout *upb_msglayout_frominit_v1(
+    const upb_msglayout_msginit_v1 *init, upb_alloc *a);
+void upb_msglayout_uninit_v1(upb_msglayout *layout, upb_alloc *a);
+
 UPB_END_EXTERN_C
 
 #endif /* UPB_MSG_H_ */
+
+UPB_BEGIN_EXTERN_C
+
+bool upb_decode(upb_stringview buf, void *msg,
+                const upb_msglayout_msginit_v1 *l, upb_env *env);
+
+UPB_END_EXTERN_C
+
+#endif  /* UPB_DECODE_H_ */
+/*
+** structs.int.h: structures definitions that are internal to upb.
+*/
+
+#ifndef UPB_STRUCTS_H_
+#define UPB_STRUCTS_H_
+
+struct upb_array {
+  upb_fieldtype_t type;
+  uint8_t element_size;
+  void *data;   /* Each element is element_size. */
+  size_t len;   /* Measured in elements. */
+  size_t size;  /* Measured in elements. */
+  upb_alloc *alloc;
+};
+
+#endif  /* UPB_STRUCTS_H_ */
+
+/*
+** This file contains definitions of structs that should be considered private
+** and NOT stable across versions of upb.
+**
+** The only reason they are declared here and not in .c files is to allow upb
+** and the application (if desired) to embed statically-initialized instances
+** of structures like defs.
+**
+** If you include this file, all guarantees of ABI compatibility go out the
+** window!  Any code that includes this file needs to recompile against the
+** exact same version of upb that they are linking against.
+**
+** You also need to recompile if you change the value of the UPB_DEBUG_REFS
+** flag.
+*/
+
+
+#ifndef UPB_STATICINIT_H_
+#define UPB_STATICINIT_H_
+
+#ifdef __cplusplus
+/* Because of how we do our typedefs, this header can't be included from C++. */
+#error This file cannot be included from C++
+#endif
+
+/* upb_refcounted *************************************************************/
+
+
+/* upb_def ********************************************************************/
+
+struct upb_def {
+  upb_refcounted base;
+
+  const char *fullname;
+  const upb_filedef* file;
+  char type;  /* A upb_deftype_t (char to save space) */
+
+  /* Used as a flag during the def's mutable stage.  Must be false unless
+   * it is currently being used by a function on the stack.  This allows
+   * us to easily determine which defs were passed into the function's
+   * current invocation. */
+  bool came_from_user;
+};
+
+#define UPB_DEF_INIT(name, type, vtbl, refs, ref2s) \
+    { UPB_REFCOUNT_INIT(vtbl, refs, ref2s), name, NULL, type, false }
+
+
+/* upb_fielddef ***************************************************************/
+
+struct upb_fielddef {
+  upb_def base;
+
+  union {
+    int64_t sint;
+    uint64_t uint;
+    double dbl;
+    float flt;
+    void *bytes;
+  } defaultval;
+  union {
+    const upb_msgdef *def;  /* If !msg_is_symbolic. */
+    char *name;             /* If msg_is_symbolic. */
+  } msg;
+  union {
+    const upb_def *def;  /* If !subdef_is_symbolic. */
+    char *name;          /* If subdef_is_symbolic. */
+  } sub;  /* The msgdef or enumdef for this field, if upb_hassubdef(f). */
+  bool subdef_is_symbolic;
+  bool msg_is_symbolic;
+  const upb_oneofdef *oneof;
+  bool default_is_string;
+  bool type_is_set_;     /* False until type is explicitly set. */
+  bool is_extension_;
+  bool lazy_;
+  bool packed_;
+  upb_intfmt_t intfmt;
+  bool tagdelim;
+  upb_fieldtype_t type_;
+  upb_label_t label_;
+  uint32_t number_;
+  uint32_t selector_base;  /* Used to index into a upb::Handlers table. */
+  uint32_t index_;
+};
+
+extern const struct upb_refcounted_vtbl upb_fielddef_vtbl;
+
+#define UPB_FIELDDEF_INIT(label, type, intfmt, tagdelim, is_extension, lazy,   \
+                          packed, name, num, msgdef, subdef, selector_base,    \
+                          index, defaultval, refs, ref2s)                      \
+  {                                                                            \
+    UPB_DEF_INIT(name, UPB_DEF_FIELD, &upb_fielddef_vtbl, refs, ref2s),        \
+        defaultval, {msgdef}, {subdef}, NULL, false, false,                    \
+        type == UPB_TYPE_STRING || type == UPB_TYPE_BYTES, true, is_extension, \
+        lazy, packed, intfmt, tagdelim, type, label, num, selector_base, index \
+  }
+
+
+/* upb_msgdef *****************************************************************/
+
+struct upb_msgdef {
+  upb_def base;
+
+  size_t selector_count;
+  uint32_t submsg_field_count;
+
+  /* Tables for looking up fields by number and name. */
+  upb_inttable itof;  /* int to field */
+  upb_strtable ntof;  /* name to field/oneof */
+
+  /* Is this a map-entry message? */
+  bool map_entry;
+
+  /* Whether this message has proto2 or proto3 semantics. */
+  upb_syntax_t syntax;
+
+  /* TODO(haberman): proper extension ranges (there can be multiple). */
+};
+
+extern const struct upb_refcounted_vtbl upb_msgdef_vtbl;
+
+/* TODO: also support static initialization of the oneofs table. This will be
+ * needed if we compile in descriptors that contain oneofs. */
+#define UPB_MSGDEF_INIT(name, selector_count, submsg_field_count, itof, ntof, \
+                        map_entry, syntax, refs, ref2s)                       \
+  {                                                                           \
+    UPB_DEF_INIT(name, UPB_DEF_MSG, &upb_fielddef_vtbl, refs, ref2s),         \
+        selector_count, submsg_field_count, itof, ntof, map_entry, syntax     \
+  }
+
+
+/* upb_enumdef ****************************************************************/
+
+struct upb_enumdef {
+  upb_def base;
+
+  upb_strtable ntoi;
+  upb_inttable iton;
+  int32_t defaultval;
+};
+
+extern const struct upb_refcounted_vtbl upb_enumdef_vtbl;
+
+#define UPB_ENUMDEF_INIT(name, ntoi, iton, defaultval, refs, ref2s) \
+  { UPB_DEF_INIT(name, UPB_DEF_ENUM, &upb_enumdef_vtbl, refs, ref2s), ntoi,    \
+    iton, defaultval }
+
+
+/* upb_oneofdef ***************************************************************/
+
+struct upb_oneofdef {
+  upb_refcounted base;
+
+  uint32_t index;  /* Index within oneofs. */
+  const char *name;
+  upb_strtable ntof;
+  upb_inttable itof;
+  const upb_msgdef *parent;
+};
+
+extern const struct upb_refcounted_vtbl upb_oneofdef_vtbl;
+
+#define UPB_ONEOFDEF_INIT(name, ntof, itof, refs, ref2s) \
+  { UPB_REFCOUNT_INIT(&upb_oneofdef_vtbl, refs, ref2s), 0, name, ntof, itof }
+
+
+/* upb_symtab *****************************************************************/
+
+struct upb_symtab {
+  upb_refcounted base;
+
+  upb_strtable symtab;
+};
+
+struct upb_filedef {
+  upb_refcounted base;
+
+  const char *name;
+  const char *package;
+  const char *phpprefix;
+  const char *phpnamespace;
+  upb_syntax_t syntax;
+
+  upb_inttable defs;
+  upb_inttable deps;
+};
+
+extern const struct upb_refcounted_vtbl upb_filedef_vtbl;
+
+#endif  /* UPB_STATICINIT_H_ */
+/*
+** upb_encode: parsing into a upb_msg using a upb_msglayout.
+*/
+
+#ifndef UPB_ENCODE_H_
+#define UPB_ENCODE_H_
+
+
+UPB_BEGIN_EXTERN_C
+
+char *upb_encode(const void *msg, const upb_msglayout_msginit_v1 *l,
+                 upb_env *env, size_t *size);
+
+UPB_END_EXTERN_C
+
+#endif  /* UPB_ENCODE_H_ */
 /*
 ** upb::descriptor::Reader (upb_descreader)
 **
@@ -8290,21 +8399,9 @@ UPB_INLINE void upb_pbdecoder_unpackdispatch(uint64_t dispatch, uint64_t *ofs,
 extern "C" {
 #endif
 
-/* A list of types as they are encoded on-the-wire. */
-typedef enum {
-  UPB_WIRE_TYPE_VARINT      = 0,
-  UPB_WIRE_TYPE_64BIT       = 1,
-  UPB_WIRE_TYPE_DELIMITED   = 2,
-  UPB_WIRE_TYPE_START_GROUP = 3,
-  UPB_WIRE_TYPE_END_GROUP   = 4,
-  UPB_WIRE_TYPE_32BIT       = 5
-} upb_wiretype_t;
-
 #define UPB_MAX_WIRE_TYPE 5
 
-/* The maximum number of bytes that it takes to encode a 64-bit varint.
- * Note that with a better encoding this could be 9 (TODO: write up a
- * wiki document about this). */
+/* The maximum number of bytes that it takes to encode a 64-bit varint. */
 #define UPB_PB_VARINT_MAX_LEN 10
 
 /* Array of the "native" (ie. non-packed-repeated) wire type for the given a

src/google/protobuf/test_messages_proto3.proto

@@ -215,6 +215,9 @@ message TestAllTypesProto3 {
   int32 field__Name16 = 416;
   int32 field_name17__ = 417;
   int32 Field_name18__ = 418;
+
+  // Reserved for testing unknown fields
+  reserved 501 to 510;
 }
 
 message ForeignMessage {