SchemaLoader -- lets you load and use schemas that aren't compiled in.

c++/Makefile.am

@@ -71,6 +71,7 @@ includecapnp_HEADERS =                                         \
   src/capnproto/list.h                                         \
   src/capnproto/message.h                                      \
   src/capnproto/schema.h                                       \
+  src/capnproto/schema-loader.h                                \
   src/capnproto/dynamic.h                                      \
   src/capnproto/stringify.h                                    \
   src/capnproto/io.h                                           \
@@ -100,6 +101,7 @@ libcapnproto_a_SOURCES=                                        \
   src/capnproto/list.c++                                       \
   src/capnproto/message.c++                                    \
   src/capnproto/schema.c++                                     \
+  src/capnproto/schema-loader.c++                              \
   src/capnproto/dynamic.c++                                    \
   src/capnproto/stringify.c++                                  \
   src/capnproto/io.c++                                         \
@@ -146,6 +148,7 @@ capnproto_test_SOURCES =                                       \
   src/capnproto/layout-test.c++                                \
   src/capnproto/message-test.c++                               \
   src/capnproto/schema-test.c++                                \
+  src/capnproto/schema-loader-test.c++                         \
   src/capnproto/dynamic-test.c++                               \
   src/capnproto/stringify-test.c++                             \
   src/capnproto/encoding-test.c++                              \

c++/src/capnproto/dynamic.c++

@@ -282,13 +282,6 @@ void DynamicUnion::Builder::setObjectDiscriminant(StructSchema::Member member) {
 
 // =======================================================================================
 
-void DynamicStruct::Reader::verifySchema(StructSchema expected) {
-  PRECOND(schema == expected, "Type mismatch when using DynamicStruct::Reader::as().");
-}
-void DynamicStruct::Builder::verifySchema(StructSchema expected) {
-  PRECOND(schema == expected, "Type mismatch when using DynamicStruct::Builder::as().");
-}
-
 DynamicValue::Reader DynamicStruct::Reader::get(StructSchema::Member member) {
   PRECOND(member.getContainingStruct() == schema, "`member` is not a member of this struct.");
   return getImpl(reader, member);
@@ -1320,15 +1313,6 @@ DynamicList::Reader DynamicList::Builder::asReader() {
   return DynamicList::Reader(schema, builder.asReader());
 }
 
-void DynamicList::Reader::verifySchema(ListSchema expectedSchema) {
-  PRECOND(schema == expectedSchema,
-          "Type mismatch when using DynamicList::Reader::as().");
-}
-void DynamicList::Builder::verifySchema(ListSchema expectedSchema) {
-  PRECOND(schema == expectedSchema,
-          "Type mismatch when using DynamicList::Reader::as().");
-}
-
 // =======================================================================================
 
 namespace {

c++/src/capnproto/dynamic.h

@@ -275,8 +275,6 @@ private:
   inline Reader(StructSchema schema, internal::StructReader reader)
       : schema(schema), reader(reader) {}
 
-  void verifySchema(StructSchema expected);
-
   static DynamicValue::Reader getImpl(internal::StructReader reader, StructSchema::Member member);
 
   template <typename T>
@@ -357,8 +355,6 @@ private:
   inline Builder(StructSchema schema, internal::StructBuilder builder)
       : schema(schema), builder(builder) {}
 
-  void verifySchema(StructSchema expected);
-
   static DynamicValue::Builder getImpl(
       internal::StructBuilder builder, StructSchema::Member member);
   static DynamicStruct::Builder getObjectImpl(
@@ -425,8 +421,6 @@ private:
 
   Reader(ListSchema schema, internal::ListReader reader): schema(schema), reader(reader) {}
 
-  void verifySchema(ListSchema expectedSchema);
-
   template <typename T>
   friend struct internal::PointerHelpers;
   friend struct DynamicStruct;
@@ -469,8 +463,6 @@ private:
 
   Builder(ListSchema schema, internal::ListBuilder builder): schema(schema), builder(builder) {}
 
-  void verifySchema(ListSchema expectedSchema);
-
   template <typename T>
   friend struct internal::PointerHelpers;
   friend struct DynamicStruct;
@@ -880,14 +872,14 @@ template <typename T>
 typename T::Reader DynamicStruct::Reader::as() {
   static_assert(kind<T>() == Kind::STRUCT,
                 "DynamicStruct::Reader::as<T>() can only convert to struct types.");
-  verifySchema(Schema::from<T>());
+  schema.requireUsableAs<T>();
   return typename T::Reader(reader);
 }
 template <typename T>
 typename T::Builder DynamicStruct::Builder::as() {
   static_assert(kind<T>() == Kind::STRUCT,
                 "DynamicStruct::Builder::as<T>() can only convert to struct types.");
-  verifySchema(Schema::from<T>());
+  schema.requireUsableAs<T>();
   return typename T::Builder(builder);
 }
 
@@ -910,14 +902,14 @@ template <typename T>
 typename T::Reader DynamicList::Reader::as() {
   static_assert(kind<T>() == Kind::LIST,
                 "DynamicStruct::Reader::as<T>() can only convert to list types.");
-  verifySchema(Schema::from<T>());
+  schema.requireUsableAs<T>();
   return typename T::Reader(reader);
 }
 template <typename T>
 typename T::Builder DynamicList::Builder::as() {
   static_assert(kind<T>() == Kind::LIST,
                 "DynamicStruct::Builder::as<T>() can only convert to list types.");
-  verifySchema(Schema::from<T>());
+  schema.requireUsableAs<T>();
   return typename T::Builder(builder);
 }
 

c++/src/capnproto/encoding-test.c++

@@ -1307,6 +1307,34 @@ TEST(Encoding, ZeroOldObject) {
   checkTestMessageAllZero(oldSub2);
 }
 
+TEST(Encoding, Has) {
+  MallocMessageBuilder builder;
+
+  auto root = builder.initRoot<TestAllTypes>();
+
+  EXPECT_FALSE(root.hasTextField());
+  EXPECT_FALSE(root.hasDataField());
+  EXPECT_FALSE(root.hasStructField());
+  EXPECT_FALSE(root.hasInt32List());
+
+  EXPECT_FALSE(root.asReader().hasTextField());
+  EXPECT_FALSE(root.asReader().hasDataField());
+  EXPECT_FALSE(root.asReader().hasStructField());
+  EXPECT_FALSE(root.asReader().hasInt32List());
+
+  initTestMessage(root);
+
+  EXPECT_TRUE(root.hasTextField());
+  EXPECT_TRUE(root.hasDataField());
+  EXPECT_TRUE(root.hasStructField());
+  EXPECT_TRUE(root.hasInt32List());
+
+  EXPECT_TRUE(root.asReader().hasTextField());
+  EXPECT_TRUE(root.asReader().hasDataField());
+  EXPECT_TRUE(root.asReader().hasStructField());
+  EXPECT_TRUE(root.asReader().hasInt32List());
+}
+
 }  // namespace
 }  // namespace internal
 }  // namespace capnproto

c++/src/capnproto/exception.c++

@@ -66,10 +66,21 @@ Exception::Exception(const Exception& other) noexcept
     : file(other.file), line(other.line), nature(other.nature), durability(other.durability),
       description(str(other.description)), traceCount(other.traceCount) {
   memcpy(trace, other.trace, sizeof(trace[0]) * traceCount);
+
+  if (other.context != nullptr) {
+    context = heap<Context>(**other.context);
+  }
 }
 
 Exception::~Exception() noexcept {}
 
+Exception::Context::Context(const Context& other) noexcept
+    : file(other.file), line(other.line), description(str(other.description)) {
+  if (other.next != nullptr) {
+    next = heap<Context>(**other.next);
+  }
+}
+
 void Exception::wrapContext(const char* file, int line, Array<char>&& description) {
   context = heap<Context>(file, line, move(description), move(context));
 }

c++/src/capnproto/exception.h

@@ -65,6 +65,7 @@ public:
   Exception(Nature nature, Durability durability, const char* file, int line,
             Array<char> description = nullptr) noexcept;
   Exception(const Exception& other) noexcept;
+  Exception(Exception&& other) = default;
   ~Exception() noexcept;
 
   const char* getFile() const { return file; }
@@ -83,6 +84,7 @@ public:
 
     Context(const char* file, int line, Array<char>&& description, Maybe<Own<Context>>&& next)
         : file(file), line(line), description(move(description)), next(move(next)) {}
+    Context(const Context& other) noexcept;
   };
 
   inline Maybe<const Context&> getContext() const {
@@ -125,6 +127,7 @@ class ExceptionCallback {
 
 public:
   ExceptionCallback();
+  CAPNPROTO_DISALLOW_COPY(ExceptionCallback);
   virtual ~ExceptionCallback();
 
   virtual void onRecoverableException(Exception&& exception);
@@ -160,6 +163,7 @@ public:
 
   public:
     ScopedRegistration(ExceptionCallback& callback);
+    CAPNPROTO_DISALLOW_COPY(ScopedRegistration);
     ~ScopedRegistration();
 
     inline ExceptionCallback& getCallback() { return callback; }

c++/src/capnproto/generated-header-support.h

@@ -141,6 +141,9 @@ struct RawSchema {
   struct MemberInfo {
     uint16_t unionIndex;  // 0 = not in a union, >0 = parent union's index + 1
     uint16_t index;       // index of the member
+
+    MemberInfo() = default;
+    inline MemberInfo(uint16_t unionIndex, uint16_t index): unionIndex(unionIndex), index(index) {}
   };
 
   const MemberInfo* membersByName;
@@ -150,6 +153,11 @@ struct RawSchema {
   uint32_t dependencyCount;
   uint32_t memberCount;
   // Sizes of above tables.
+
+  const RawSchema* canCastTo;
+  // Points to the RawSchema of a compiled-in type to which it is safe to cast any DynamicValue
+  // with this schema.  This is null for all compiled-in types; it is only set by SchemaLoader on
+  // dynamically-loaded types.
 };
 
 template <typename T>

c++/src/capnproto/layout.c++

@@ -1956,7 +1956,7 @@ const word* StructReader::getUncheckedPointer(WirePointerCount ptrIndex) const {
 }
 
 bool StructReader::isPointerFieldNull(WirePointerCount ptrIndex) const {
-  return (pointers + ptrIndex)->isNull();
+  return ptrIndex >= pointerCount || (pointers + ptrIndex)->isNull();
 }
 
 WordCount64 StructReader::totalSize() const {

c++/src/capnproto/logging.h

@@ -156,6 +156,7 @@ public:
   class Context: public ExceptionCallback {
   public:
     Context();
+    CAPNPROTO_DISALLOW_COPY(Context);
     virtual ~Context();
     virtual void addTo(Exception& exception) = 0;
 
@@ -171,20 +172,16 @@ public:
   template <typename Func>
   class ContextImpl: public Context {
   public:
-    inline ContextImpl(Func&& func): func(capnproto::move(func)) {}
+    inline ContextImpl(Func& func): func(func) {}
+    CAPNPROTO_DISALLOW_COPY(ContextImpl);
 
     void addTo(Exception& exception) override {
       func(exception);
     }
   private:
-    Func func;
+    Func& func;
   };
 
-  template <typename Func>
-  static ContextImpl<RemoveReference<Func>> context(Func&& func) {
-    return ContextImpl<RemoveReference<Func>>(capnproto::forward<Func>(func));
-  }
-
   template <typename... Params>
   static void addContextTo(Exception& exception, const char* file,
                            int line, const char* macroArgs, Params&&... params);
@@ -271,11 +268,11 @@ ArrayPtr<const char> operator*(const Stringifier&, Log::Severity severity);
   } while (false)
 
 #define CONTEXT(...) \
-  auto _capnpLoggingContext = ::capnproto::Log::context( \
-      [&](::capnproto::Exception& exception) { \
+  auto _capnpContextFunc = [&](::capnproto::Exception& exception) { \
         return ::capnproto::Log::addContextTo(exception, \
             __FILE__, __LINE__, #__VA_ARGS__, ##__VA_ARGS__); \
-      })
+      }; \
+  ::capnproto::Log::ContextImpl<decltype(_capnpContextFunc)> _capnpContext(_capnpContextFunc)
 
 #ifdef NDEBUG
 #define DLOG(...) do {} while (false)

c++/src/capnproto/message.c++

@@ -212,4 +212,20 @@ ArrayPtr<word> MallocMessageBuilder::allocateSegment(uint minimumSize) {
   return arrayPtr(reinterpret_cast<word*>(result), size);
 }
 
+// -------------------------------------------------------------------
+
+FlatMessageBuilder::FlatMessageBuilder(ArrayPtr<word> array): array(array), allocated(false) {}
+FlatMessageBuilder::~FlatMessageBuilder() {}
+
+void FlatMessageBuilder::requireFilled() {
+  PRECOND(getSegmentsForOutput()[0].end() == array.end(),
+          "FlatMessageBuilder's buffer was too large.");
+}
+
+ArrayPtr<word> FlatMessageBuilder::allocateSegment(uint minimumSize) {
+  PRECOND(!allocated, "FlatMessageBuilder's buffer was not large enough.");
+  allocated = true;
+  return array;
+}
+
 }  // namespace capnproto

c++/src/capnproto/message.h

@@ -175,7 +175,7 @@ private:
 };
 
 template <typename RootType>
-static typename RootType::Reader readMessageUnchecked(const word* data);
+typename RootType::Reader readMessageUnchecked(const word* data);
 // IF THE INPUT IS INVALID, THIS MAY CRASH, CORRUPT MEMORY, CREATE A SECURITY HOLE IN YOUR APP,
 // MURDER YOUR FIRST-BORN CHILD, AND/OR BRING ABOUT ETERNAL DAMNATION ON ALL OF HUMANITY.  DO NOT
 // USE UNLESS YOU UNDERSTAND THE CONSEQUENCES.
@@ -198,12 +198,16 @@ static typename RootType::Reader readMessageUnchecked(const word* data);
 //    MyMessage::Reader reader = Message<MyMessage>::readMessageUnchecked(MyMessage::DEFAULT.words);
 //
 // To sanitize a message from an untrusted source such that it can be safely passed to
-// readMessageUnchecked(), construct a MessageBuilder whose first segment is large enough to store
-// the message, and then use MessageBuilder::setRoot() to copy the message in.  The process of
-// copying the message implicitly validates all pointers.
+// readMessageUnchecked(), use copyToUnchecked().
+
+template <typename Reader>
+void copyToUnchecked(Reader&& reader, ArrayPtr<word> uncheckedBuffer);
+// Copy the content of the given reader into the given buffer, such that it can safely be passed to
+// readMessageUnchecked().  The buffer's size must be exactly reader.totalSizeInWords() + 1,
+// otherwise an exception will be thrown.
 
 template <typename Type>
-static typename Type::Reader defaultValue();
+typename Type::Reader defaultValue();
 // Get a default instance of the given struct or list type.
 //
 // TODO(cleanup):  Find a better home for this function?
@@ -296,6 +300,25 @@ private:
   std::unique_ptr<MoreSegments> moreSegments;
 };
 
+class FlatMessageBuilder: public MessageBuilder {
+  // A message builder implementation which allocates from a single flat array, throwing an
+  // exception if it runs out of space.
+
+public:
+  explicit FlatMessageBuilder(ArrayPtr<word> array);
+  CAPNPROTO_DISALLOW_COPY(FlatMessageBuilder);
+  virtual ~FlatMessageBuilder();
+
+  void requireFilled();
+  // Throws an exception if the flat array is not exactly full.
+
+  virtual ArrayPtr<word> allocateSegment(uint minimumSize) override;
+
+private:
+  ArrayPtr<word> array;
+  bool allocated;
+};
+
 // =======================================================================================
 // implementation details
 
@@ -333,6 +356,13 @@ typename RootType::Reader readMessageUnchecked(const word* data) {
   return typename RootType::Reader(internal::StructReader::readRootUnchecked(data));
 }
 
+template <typename Reader>
+void copyToUnchecked(Reader&& reader, ArrayPtr<word> uncheckedBuffer) {
+  FlatMessageBuilder builder(uncheckedBuffer);
+  builder.setRoot(capnproto::forward<Reader>(reader));
+  builder.requireFilled();
+}
+
 template <typename Type>
 static typename Type::Reader defaultValue() {
   // TODO(soon):  Correctly handle lists.  Maybe primitives too?

c++/src/capnproto/schema-loader-test.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#define CAPNPROTO_PRIVATE
+#include "schema-loader.h"
+#include <gtest/gtest.h>
+#include "test-util.h"
+#include "logging.h"
+
+namespace capnproto {
+namespace internal {
+namespace {
+
+TEST(SchemaLoader, Load) {
+  SchemaLoader loader;
+
+  Schema struct32Schema = loader.load(Schema::from<test::TestLists::Struct32>().getProto());
+
+  auto nativeSchema = Schema::from<test::TestLists>();
+  Schema testListsSchema = loader.load(nativeSchema.getProto());
+
+  Schema struct8Schema = loader.load(Schema::from<test::TestLists::Struct8>().getProto());
+  Schema structPSchema = loader.load(Schema::from<test::TestLists::StructP>().getProto());
+
+  EXPECT_STREQ(nativeSchema.getProto().debugString().cStr(),
+               testListsSchema.getProto().debugString().cStr());
+
+  EXPECT_FALSE(testListsSchema == nativeSchema);
+  EXPECT_FALSE(struct32Schema == Schema::from<test::TestLists::Struct32>());
+  EXPECT_FALSE(struct8Schema == Schema::from<test::TestLists::Struct8>());
+  EXPECT_FALSE(structPSchema == Schema::from<test::TestLists::StructP>());
+
+  EXPECT_TRUE(testListsSchema.getDependency(typeId<test::TestLists::Struct32>()) == struct32Schema);
+  EXPECT_TRUE(testListsSchema.getDependency(typeId<test::TestLists::Struct8>()) == struct8Schema);
+  EXPECT_TRUE(testListsSchema.getDependency(typeId<test::TestLists::StructP>()) == structPSchema);
+
+  auto struct16Schema = testListsSchema.getDependency(typeId<test::TestLists::Struct16>());
+  EXPECT_EQ(0u, struct16Schema.getProto().getBody().getStructNode().getMembers().size());
+}
+
+TEST(SchemaLoader, Use) {
+  SchemaLoader loader;
+
+  StructSchema schema = loader.load(Schema::from<TestAllTypes>().getProto()).asStruct();
+
+  // Also have to load TestEnum.
+  loader.load(Schema::from<TestEnum>().getProto());
+
+  {
+    MallocMessageBuilder builder;
+    auto root = builder.getRoot<DynamicStruct>(schema);
+
+    initDynamicTestMessage(root);
+    checkDynamicTestMessage(root.asReader());
+
+    // Can't convert to TestAllTypes because we didn't use loadCompiledTypeAndDependencies().
+    EXPECT_ANY_THROW(root.as<TestAllTypes>());
+
+    // But if we reinterpret the raw bytes, it works.
+    checkTestMessage(builder.getRoot<TestAllTypes>());
+  }
+
+  loader.loadCompiledTypeAndDependencies<TestAllTypes>();
+
+  {
+    MallocMessageBuilder builder;
+    auto root = builder.getRoot<DynamicStruct>(schema);
+
+    initDynamicTestMessage(root);
+
+    // Now we can actually cast.
+    checkTestMessage(root.as<TestAllTypes>());
+  }
+
+  // Let's also test TestListDefaults, but as we do so, let's load the compiled types first, to
+  // make sure the opposite order works.
+
+  loader.loadCompiledTypeAndDependencies<TestListDefaults>();
+  StructSchema testListsSchema = loader.get(typeId<TestListDefaults>()).asStruct();
+  EXPECT_TRUE(testListsSchema != Schema::from<TestListDefaults>());
+
+  {
+    MallocMessageBuilder builder;
+    auto root = builder.getRoot<DynamicStruct>(testListsSchema);
+
+    initDynamicTestLists(root);
+    checkDynamicTestLists(root.asReader());
+
+    checkTestMessage(root.as<TestListDefaults>());
+  }
+
+  EXPECT_TRUE(loader.load(Schema::from<TestListDefaults>().getProto()) == testListsSchema);
+
+  {
+    MallocMessageBuilder builder;
+    auto root = builder.getRoot<DynamicStruct>(testListsSchema);
+
+    initDynamicTestLists(root);
+    checkTestMessage(root.as<TestListDefaults>());
+  }
+
+  // Finally, let's test some unions.
+  StructSchema unionSchema = loader.load(Schema::from<TestUnion>().getProto()).asStruct();
+  {
+    MallocMessageBuilder builder;
+    auto root = builder.getRoot<DynamicStruct>(unionSchema);
+
+    root.get("union0").as<DynamicUnion>().set("u0f1s16", 123);
+    root.get("union1").as<DynamicUnion>().set("u1f0sp", "hello");
+
+    auto reader = builder.getRoot<TestUnion>().asReader();
+    EXPECT_EQ(123, reader.getUnion0().getU0f1s16());
+    EXPECT_EQ("hello", reader.getUnion1().getU1f0sp());
+  }
+}
+
+template <typename T>
+Schema loadUnderAlternateTypeId(SchemaLoader& loader, uint64_t id) {
+  MallocMessageBuilder schemaBuilder;
+  schemaBuilder.setRoot(Schema::from<T>().getProto());
+  auto root = schemaBuilder.getRoot<schema::Node>();
+  root.setId(id);
+
+  if (root.getBody().which() == schema::Node::Body::STRUCT_NODE) {
+    // If the struct contains any self-referential members, change their type IDs as well.
+    auto members = root.getBody().getStructNode().getMembers();
+    for (auto member: members) {
+      if (member.getBody().which() == schema::StructNode::Member::Body::FIELD_MEMBER) {
+        auto type = member.getBody().getFieldMember().getType().getBody();
+        if (type.which() == schema::Type::Body::STRUCT_TYPE &&
+            type.getStructType() == typeId<T>()) {
+          type.setStructType(id);
+        }
+      }
+    }
+  }
+
+  return loader.load(root);
+}
+
+TEST(SchemaLoader, Upgrade) {
+  SchemaLoader loader;
+
+  loader.loadCompiledTypeAndDependencies<test::TestOldVersion>();
+
+  StructSchema schema = loader.get(typeId<test::TestOldVersion>()).asStruct();
+
+  EXPECT_STREQ(Schema::from<test::TestOldVersion>().getProto().debugString().cStr(),
+               schema.getProto().debugString().cStr());
+
+  loadUnderAlternateTypeId<test::TestNewVersion>(loader, typeId<test::TestOldVersion>());
+
+  // The new version replaced the old.
+  EXPECT_STREQ(Schema::from<test::TestNewVersion>().getProto().getDisplayName(),
+               schema.getProto().getDisplayName());
+
+  // But it is still usable as the old version.
+  schema.requireUsableAs<test::TestOldVersion>();
+}
+
+TEST(SchemaLoader, Downgrade) {
+  SchemaLoader loader;
+
+  loader.loadCompiledTypeAndDependencies<test::TestNewVersion>();
+
+  StructSchema schema = loader.get(typeId<test::TestNewVersion>()).asStruct();
+
+  EXPECT_STREQ(Schema::from<test::TestNewVersion>().getProto().debugString().cStr(),
+               schema.getProto().debugString().cStr());
+
+  loadUnderAlternateTypeId<test::TestOldVersion>(loader, typeId<test::TestNewVersion>());
+
+  // We kept the new version, because the replacement was older.
+  EXPECT_STREQ(Schema::from<test::TestNewVersion>().getProto().getDisplayName(),
+               schema.getProto().getDisplayName());
+  schema.requireUsableAs<test::TestNewVersion>();
+}
+
+TEST(SchemaLoader, Incompatible) {
+  SchemaLoader loader;
+  loader.loadCompiledTypeAndDependencies<test::TestListDefaults>();
+  EXPECT_ANY_THROW(
+      loadUnderAlternateTypeId<test::TestAllTypes>(loader, typeId<test::TestListDefaults>()));
+}
+
+TEST(SchemaLoader, Enumerate) {
+  SchemaLoader loader;
+  loader.loadCompiledTypeAndDependencies<TestAllTypes>();
+  auto list = loader.getAllLoaded();
+
+  ASSERT_EQ(2u, list.size());
+  if (list[0] == loader.get(typeId<TestAllTypes>())) {
+    EXPECT_TRUE(list[1] == loader.get(typeId<TestEnum>()));
+  } else {
+    EXPECT_TRUE(list[0] == loader.get(typeId<TestEnum>()));
+    EXPECT_TRUE(list[1] == loader.get(typeId<TestAllTypes>()));
+  }
+}
+
+// TODO(test):  More extensively test upgrade/downgrade checks.
+
+}  // namespace
+}  // namespace internal
+}  // namespace capnproto

c++/src/capnproto/schema-loader.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef CAPNPROTO_SCHEMA_LOADER_H_
+#define CAPNPROTO_SCHEMA_LOADER_H_
+
+#include "schema.h"
+
+namespace capnproto {
+
+class SchemaLoader {
+public:
+  SchemaLoader();
+  ~SchemaLoader();
+  CAPNPROTO_DISALLOW_COPY(SchemaLoader);
+
+  Schema get(uint64_t id) const;
+  // Gets the schema for the given ID, throwing an exception if it isn't present.
+  //
+  // The returned schema may be invalidated if load() is called with a new schema for the same ID.
+  // In general, you should not call load() while a schema from this loader is in-use.
+
+  Maybe<Schema> tryGet(uint64_t id) const;
+  // Like get() but doesn't throw.
+
+  Schema load(schema::Node::Reader reader);
+  // Loads the given schema node.  Validates the node and throws an exception if invalid.  This
+  // makes a copy of the schema, so the object passed in can be destroyed after this returns.
+  //
+  // If the node has any dependencies which are not already loaded, they will be initialized as
+  // stubs -- empty schemas of whichever kind is expected.
+  //
+  // If another schema for the given reader has already been seen, the loader will inspect both
+  // schemas to determine which one is newer, and use that that one.  If the two versions are
+  // found to be incompatible, an exception is thrown.  If the two versions differ but are
+  // compatible and the loader cannot determine which is newer (e.g., the only changes are renames),
+  // the existing schema will be preferred.  Note that in any case, the loader will end up keeping
+  // around copies of both schemas, so you shouldn't repeatedly reload schemas into the same loader.
+  //
+  // The following properties of the schema node are validated:
+  // - Struct size and preferred list encoding are valid and consistent.
+  // - Struct members are fields or unions.
+  // - Union members are fields.
+  // - Field offsets are in-bounds.
+  // - Ordinals and codeOrders are sequential starting from zero.
+  // - Values are of the right union case to match their types.
+  //
+  // You should assume anything not listed above is NOT validated.  In particular, things that are
+  // not validated now, but could be in the future, include but are not limited to:
+  // - Names.
+  // - Annotation values.  (This is hard because the annotation declaration is not always
+  //   available.)
+  // - Content of default/constant values of pointer type.  (Validating these would require knowing
+  //   their schema, but even if the schemas are available at validation time, they could be
+  //   updated by a subsequent load(), invalidating existing values.  Instead, these values are
+  //   validated at the time they are used, as usual for Cap'n Proto objects.)
+  //
+  // Also note that unknown types are not considered invalid.  Instead, the dynamic API returns
+  // a DynamicValue with type UNKNOWN for these.
+
+  template <typename T>
+  void loadCompiledTypeAndDependencies();
+  // Load the schema for the given compiled-in type and all of its dependencies.
+  //
+  // If you want to be able to cast a DynamicValue built from this SchemaLoader to the compiled-in
+  // type using as<T>(), you must call this method before constructing the DynamicValue.  Otherwise,
+  // as<T>() will throw an exception complaining about type mismatch.
+
+  Array<Schema> getAllLoaded() const;
+  // Get a complete list of all loaded schema nodes.  It is particularly useful to call this after
+  // loadCompiledTypeAndDependencies<T>() in order to get a flat list of all of T's transitive
+  // dependencies.
+
+private:
+  class Validator;
+  class CompatibilityChecker;
+  class Impl;
+  Own<Impl> impl;
+
+  void loadNative(const internal::RawSchema* nativeSchema);
+};
+
+template <typename T>
+inline void SchemaLoader::loadCompiledTypeAndDependencies() {
+  loadNative(&internal::rawSchema<T>());
+}
+
+}  // namespace capnproto
+
+#endif  // CAPNPROTO_SCHEMA_LOADER_H_

c++/src/capnproto/schema.c++

@@ -76,6 +76,12 @@ InterfaceSchema Schema::asInterface() const {
   return InterfaceSchema(raw);
 }
 
+void Schema::requireUsableAs(const internal::RawSchema* expected) {
+  PRECOND(raw == expected ||
+          (raw != nullptr && expected != nullptr && raw->canCastTo == expected),
+          "This schema is not compatible with the requested native type.");
+}
+
 // =======================================================================================
 
 namespace {
@@ -285,4 +291,10 @@ ListSchema ListSchema::getListElementType() const {
   return ListSchema(elementType, nestingDepth - 1, elementSchema);
 }
 
+void ListSchema::requireUsableAs(ListSchema expected) {
+  PRECOND(elementType == expected.elementType && nestingDepth == expected.nestingDepth,
+          "This schema is not compatible with the requested native type.");
+  elementSchema.requireUsableAs(expected.elementSchema.raw);
+}
+
 }  // namespace capnproto

c++/src/capnproto/schema.h

@@ -75,6 +75,14 @@ public:
   // you want to check if two Schemas represent the same type (but possibly different versions of
   // it), compare their IDs instead.
 
+  template <typename T>
+  void requireUsableAs();
+  // Throws an exception if a value with this Schema cannot safely be cast to a native value of
+  // the given type.  This passes if either:
+  // - *this == from<T>()
+  // - This schema was loaded with SchemaLoader, the type ID matches typeId<T>(), and
+  //   loadCompiledTypeAndDependencies<T>() was called on the SchemaLoader.
+
 private:
   const internal::RawSchema* raw;
 
@@ -84,9 +92,13 @@ private:
     return Schema(&internal::rawSchema<T>());
   }
 
+  void requireUsableAs(const internal::RawSchema* expected);
+
   friend class StructSchema;
   friend class EnumSchema;
   friend class InterfaceSchema;
+  friend class ListSchema;
+  friend class SchemaLoader;
 };
 
 // -------------------------------------------------------------------
@@ -356,6 +368,9 @@ public:
   inline bool operator==(const ListSchema& other) const;
   inline bool operator!=(const ListSchema& other) const { return !(*this == other); }
 
+  template <typename T>
+  void requireUsableAs();
+
 private:
   schema::Type::Body::Which elementType;
   uint8_t nestingDepth;  // 0 for T, 1 for List(T), 2 for List(List(T)), ...
@@ -375,6 +390,8 @@ private:
     return FromImpl<T>::get();
   }
 
+  void requireUsableAs(ListSchema expected);
+
   friend class Schema;
 };
 
@@ -396,6 +413,11 @@ template <> inline schema::Type::Body::Which Schema::from<double>() { return sch
 template <> inline schema::Type::Body::Which Schema::from<Text>() { return schema::Type::Body::TEXT_TYPE; }
 template <> inline schema::Type::Body::Which Schema::from<Data>() { return schema::Type::Body::DATA_TYPE; }
 
+template <typename T>
+inline void Schema::requireUsableAs() {
+  requireUsableAs(&internal::rawSchema<T>());
+}
+
 inline bool StructSchema::Member::operator==(const Member& other) const {
   return parent == other.parent && unionIndex == other.unionIndex && index == other.index;
 }
@@ -429,6 +451,13 @@ inline bool ListSchema::operator==(const ListSchema& other) const {
       elementSchema == other.elementSchema;
 }
 
+template <typename T>
+inline void ListSchema::requireUsableAs() {
+  static_assert(kind<T>() == Kind::LIST,
+                "ListSchema::requireUsableAs<T>() requires T is a list type.");
+  requireUsableAs(Schema::from<T>());
+}
+
 template <typename T>
 struct ListSchema::FromImpl<List<T>> {
   static inline ListSchema get() { return of(Schema::from<T>()); }

c++/src/capnproto/test-util.h

@@ -27,6 +27,7 @@
 #include "test.capnp.h"
 #include <iostream>
 #include "blob.h"
+#include "dynamic.h"
 
 namespace capnproto {
 
@@ -76,6 +77,15 @@ void checkTestMessage(TestListDefaults::Reader reader);
 void checkTestMessageAllZero(TestAllTypes::Builder builder);
 void checkTestMessageAllZero(TestAllTypes::Reader reader);
 
+void initDynamicTestMessage(DynamicStruct::Builder builder);
+void initDynamicTestLists(DynamicStruct::Builder builder);
+void checkDynamicTestMessage(DynamicStruct::Builder builder);
+void checkDynamicTestLists(DynamicStruct::Builder builder);
+void checkDynamicTestMessage(DynamicStruct::Reader reader);
+void checkDynamicTestLists(DynamicStruct::Reader reader);
+void checkDynamicTestMessageAllZero(DynamicStruct::Builder builder);
+void checkDynamicTestMessageAllZero(DynamicStruct::Reader reader);
+
 }  // namespace internal
 }  // namespace capnproto
 

compiler/src/c++-header.mustache

@@ -168,6 +168,7 @@ public:
   inline {{fieldType}} get{{fieldTitleCase}}();
 {{/fieldIsPrimitive}}
 {{^fieldIsPrimitive}}
+  inline bool has{{fieldTitleCase}}();
 {{^fieldIsGenericObject}}
   inline {{fieldType}}::Reader get{{fieldTitleCase}}();
 {{/fieldIsGenericObject}}
@@ -218,6 +219,7 @@ public:
   inline void set{{fieldTitleCase}}({{fieldType}} value);
 {{/fieldIsPrimitive}}
 {{^fieldIsPrimitive}}
+  inline bool has{{fieldTitleCase}}();
 {{^fieldIsGenericObject}}
   inline {{fieldType}}::Builder get{{fieldTitleCase}}();
   inline void set{{fieldTitleCase}}({{fieldType}}::Reader other);
@@ -310,8 +312,16 @@ inline void {{typeFullName}}::Builder::set{{fieldTitleCase}}({{fieldType}} value
 }
 {{/fieldIsPrimitive}}
 {{! ------------------------------------------------------------------------------------------- }}
-{{^fieldIsGenericObject}}
 {{^fieldIsPrimitive}}
+inline bool {{typeFullName}}::Reader::has{{fieldTitleCase}}() {
+  return !_reader.isPointerFieldNull({{fieldOffset}} * ::capnproto::POINTERS);
+}
+
+inline bool {{typeFullName}}::Builder::has{{fieldTitleCase}}() {
+  return !_builder.isPointerFieldNull({{fieldOffset}} * ::capnproto::POINTERS);
+}
+
+{{^fieldIsGenericObject}}
 inline {{fieldType}}::Reader {{typeFullName}}::Reader::get{{fieldTitleCase}}() {
 {{#fieldUnion}}
   CAPNPROTO_INLINE_DPRECOND(which() == {{unionTitleCase}}::{{fieldUpperCase}},
@@ -375,7 +385,6 @@ inline {{fieldType}}::Builder {{typeFullName}}::Builder::init{{fieldTitleCase}}(
 }
 
 {{/fieldIsStruct}}
-{{/fieldIsPrimitive}}
 {{/fieldIsGenericObject}}
 {{! ------------------------------------------------------------------------------------------- }}
 {{#fieldIsGenericObject}}
@@ -450,6 +459,7 @@ inline typename T::Builder {{typeFullName}}::Builder::init{{fieldTitleCase}}(Par
 }
 
 {{/fieldIsGenericObject}}
+{{/fieldIsPrimitive}}
 {{/typeFields}}
 {{/typeStructOrUnion}}
 {{/fileTypes}}

compiler/src/c++-source.mustache

@@ -69,7 +69,8 @@ static const ::capnproto::internal::RawSchema::MemberInfo m_{{schemaId}}[] = {
 {{/schemaMembersByName}}
 };
 const ::capnproto::internal::RawSchema s_{{schemaId}} = {
-  b_{{schemaId}}.words, d_{{schemaId}}, m_{{schemaId}}, {{schemaDependencyCount}}, {{schemaMemberCount}}
+  b_{{schemaId}}.words, d_{{schemaId}}, m_{{schemaId}},
+  {{schemaDependencyCount}}, {{schemaMemberCount}}, nullptr
 };
 {{/typeSchema}}
 {{/fileTypes}}