Dynamic API mostly done... not tested. This commit actually starts embedding…

Dynamic API mostly done...  not tested.  This commit actually starts embedding the schema data in the generated code.

c++/src/capnproto/blob.h

@@ -55,6 +55,8 @@ class Data::Reader {
   // also be implicitly constructed from a NUL-terminated char*.
 
 public:
+  typedef Data Reads;
+
   inline Reader(): bytes(nullptr), size_(0) {}
   inline Reader(const char* bytes): bytes(bytes), size_(strlen(bytes)) {}
   inline Reader(char* bytes): bytes(bytes), size_(strlen(bytes)) {}
@@ -113,6 +115,8 @@ class Text::Reader: public Data::Reader {
   // such as std::string.
 
 public:
+  typedef Text Reads;
+
   inline Reader(): Data::Reader("", 0) {}
   inline Reader(const char* text): Data::Reader(text, strlen(text)) {
     CAPNPROTO_INLINE_DPRECOND(text[size()] == '\0', "Text must be NUL-terminated.");
@@ -139,6 +143,8 @@ class Data::Builder {
   // other types.
 
 public:
+  typedef Data Builds;
+
   inline Builder(): bytes(nullptr), size_(0) {}
   inline Builder(char* bytes, uint size): bytes(bytes), size_(size) {}
 
@@ -194,6 +200,8 @@ class Text::Builder: public Data::Builder {
   // so it is never necessary for the caller to do so.
 
 public:
+  typedef Text Builds;
+
   inline Builder(): Data::Builder(nulstr, 0) {}
   inline Builder(char* text, uint size): Data::Builder(text, size) { text[size] = '\0'; }
 

c++/src/capnproto/dynamic.c++

@@ -25,7 +25,7 @@
 #include "dynamic.h"
 #include "logging.h"
 #include <unordered_map>
-#include <string>
+#include <unordered_set>
 
 namespace capnproto {
 
@@ -52,17 +52,31 @@ struct SchemaPool::Impl {
   std::unordered_map<std::pair<uint64_t, Text::Reader>, schema::EnumNode::Enumerant::Reader,
                      IdTextHash>
       enumerantMap;
+  std::unordered_set<uint64_t> compiledTypeIds;
 };
 
+SchemaPool::SchemaPool(): impl(new Impl) {}
+
 SchemaPool::~SchemaPool() {
   delete impl;
 }
 
 // TODO(now):  Implement this.  Need to copy, ick.
-void add(schema::Node::Reader node) {
+void SchemaPool::add(schema::Node::Reader node) {
   FAIL_CHECK("Not implemented: copying/validating schemas.");
 }
 
+void SchemaPool::add(const internal::RawSchema& rawSchema) {
+  auto node = readMessageTrusted<schema::Node>(rawSchema.encodedNode);
+  if (impl->compiledTypeIds.insert(node.getId()).second) {
+    addNoCopy(node);
+    for (const internal::RawSchema*const* dep = rawSchema.dependencies;
+         *dep != nullptr; ++dep) {
+      add(**dep);
+    }
+  }
+}
+
 void SchemaPool::addNoCopy(schema::Node::Reader node) {
   if (impl == nullptr) {
     impl = new Impl;
@@ -79,8 +93,41 @@ void SchemaPool::addNoCopy(schema::Node::Reader node) {
   }
 }
 
-bool SchemaPool::has(uint64_t id) const {
-  return (impl != nullptr && impl->nodeMap.count(id) != 0) || (base != nullptr && base->has(id));
+Maybe<schema::Node::Reader> SchemaPool::tryGetNode(uint64_t id) const {
+  auto iter = impl->nodeMap.find(id);
+  if (iter == impl->nodeMap.end()) {
+    return nullptr;
+  } else {
+    return iter->second;
+  }
+}
+
+schema::Node::Reader SchemaPool::getNode(uint64_t id) const {
+  auto maybeNode = tryGetNode(id);
+  PRECOND(maybeNode != nullptr, "Requested node ID not found in SchemaPool", hex(id));
+  return *maybeNode;
+}
+
+schema::Node::Reader SchemaPool::getStruct(uint64_t id) const {
+  auto node = getNode(id);
+  PRECOND(node.getBody().which() == schema::Node::Body::STRUCT_NODE,
+          "Looking for a struct node, but this node ID refers to something else.",
+          node.getBody().which());
+  return node;
+}
+schema::Node::Reader SchemaPool::getEnum(uint64_t id) const {
+  auto node = getNode(id);
+  PRECOND(node.getBody().which() == schema::Node::Body::ENUM_NODE,
+          "Looking for a enum node, but this node ID refers to something else.",
+          node.getBody().which());
+  return node;
+}
+schema::Node::Reader SchemaPool::getInterface(uint64_t id) const {
+  auto node = getNode(id);
+  PRECOND(node.getBody().which() == schema::Node::Body::INTERFACE_NODE,
+          "Looking for a interface node, but this node ID refers to something else.",
+          node.getBody().which());
+  return node;
 }
 
 // =======================================================================================
@@ -141,7 +188,7 @@ internal::FieldSize elementSizeFor(schema::Type::Body::Which elementType) {
     case schema::Type::Body::ENUM_TYPE: return internal::FieldSize::TWO_BYTES;
     case schema::Type::Body::STRUCT_TYPE: return internal::FieldSize::INLINE_COMPOSITE;
     case schema::Type::Body::INTERFACE_TYPE: return internal::FieldSize::REFERENCE;
-    case schema::Type::Body::OBJECT_TYPE: FAIL_CHECK("List(Object) not supported.");
+    case schema::Type::Body::OBJECT_TYPE: FAIL_CHECK("List(Object) not supported."); break;
   }
   FAIL_CHECK("Can't get here.");
   return internal::FieldSize::VOID;
@@ -156,6 +203,35 @@ inline internal::StructSize structSizeFromSchema(schema::StructNode::Reader sche
 
 }  // namespace
 
+namespace internal {
+
+ListSchema::ListSchema(schema::Type::Reader elementType): nestingDepth(0) {
+  auto etypeBody = elementType.getBody();
+  while (etypeBody.which() == schema::Type::Body::LIST_TYPE) {
+    ++nestingDepth;
+    etypeBody = etypeBody.getListType().getBody();
+  }
+
+  this->elementType = etypeBody.which();
+
+  switch (etypeBody.which()) {
+    case schema::Type::Body::STRUCT_TYPE:
+      this->elementTypeId = etypeBody.getStructType();
+      break;
+    case schema::Type::Body::ENUM_TYPE:
+      this->elementTypeId = etypeBody.getEnumType();
+      break;
+    case schema::Type::Body::INTERFACE_TYPE:
+      this->elementTypeId = etypeBody.getInterfaceType();
+      break;
+    default:
+      this->elementTypeId = 0;
+      break;
+  }
+}
+
+}  // namespace internal
+
 // =======================================================================================
 
 schema::EnumNode::Reader DynamicEnum::getSchema() {
@@ -189,9 +265,9 @@ uint16_t DynamicEnum::asImpl(uint64_t requestedTypeId) {
 
 // =======================================================================================
 
-DynamicStruct::Reader DynamicObject::Reader::toStruct(schema::Node::Reader schema) {
+DynamicStruct::Reader DynamicObject::Reader::asStruct(schema::Node::Reader schema) {
   PRECOND(schema.getBody().which() == schema::Node::Body::STRUCT_NODE,
-          "toStruct() passed a non-struct schema.");
+          "asStruct() passed a non-struct schema.");
   if (reader.kind == internal::ObjectKind::NULL_POINTER) {
     return DynamicStruct::Reader(pool, schema, internal::StructReader());
   }
@@ -200,9 +276,9 @@ DynamicStruct::Reader DynamicObject::Reader::toStruct(schema::Node::Reader schem
   }
   return DynamicStruct::Reader(pool, schema, reader.structReader);
 }
-DynamicStruct::Builder DynamicObject::Builder::toStruct(schema::Node::Reader schema) {
+DynamicStruct::Builder DynamicObject::Builder::asStruct(schema::Node::Reader schema) {
   PRECOND(schema.getBody().which() == schema::Node::Body::STRUCT_NODE,
-          "toStruct() passed a non-struct schema.");
+          "asStruct() passed a non-struct schema.");
   if (builder.kind == internal::ObjectKind::NULL_POINTER) {
     return DynamicStruct::Builder(pool, schema, internal::StructBuilder());
   }
@@ -212,21 +288,21 @@ DynamicStruct::Builder DynamicObject::Builder::toStruct(schema::Node::Reader sch
   return DynamicStruct::Builder(pool, schema, builder.structBuilder);
 }
 
-DynamicStruct::Reader DynamicObject::Reader::toStruct(uint64_t typeId) {
-  return toStruct(pool->getStruct(typeId));
+DynamicStruct::Reader DynamicObject::Reader::asStruct(uint64_t typeId) {
+  return asStruct(pool->getStruct(typeId));
 }
-DynamicStruct::Builder DynamicObject::Builder::toStruct(uint64_t typeId) {
-  return toStruct(pool->getStruct(typeId));
+DynamicStruct::Builder DynamicObject::Builder::asStruct(uint64_t typeId) {
+  return asStruct(pool->getStruct(typeId));
 }
 
-DynamicList::Reader DynamicObject::Reader::toList(schema::Type::Reader elementType) {
-  return toList(internal::ListSchema(elementType));
+DynamicList::Reader DynamicObject::Reader::asList(schema::Type::Reader elementType) {
+  return asList(internal::ListSchema(elementType));
 }
-DynamicList::Builder DynamicObject::Builder::toList(schema::Type::Reader elementType) {
-  return toList(internal::ListSchema(elementType));
+DynamicList::Builder DynamicObject::Builder::asList(schema::Type::Reader elementType) {
+  return asList(internal::ListSchema(elementType));
 }
 
-DynamicList::Reader DynamicObject::Reader::toList(internal::ListSchema schema) {
+DynamicList::Reader DynamicObject::Reader::asList(internal::ListSchema schema) {
   if (reader.kind == internal::ObjectKind::NULL_POINTER) {
     return DynamicList::Reader(pool, schema, internal::ListReader());
   }
@@ -235,7 +311,7 @@ DynamicList::Reader DynamicObject::Reader::toList(internal::ListSchema schema) {
   }
   return DynamicList::Reader(pool, schema, reader.listReader);
 }
-DynamicList::Builder DynamicObject::Builder::toList(internal::ListSchema schema) {
+DynamicList::Builder DynamicObject::Builder::asList(internal::ListSchema schema) {
   if (builder.kind == internal::ObjectKind::NULL_POINTER) {
     return DynamicList::Builder(pool, schema, internal::ListBuilder());
   }
@@ -775,6 +851,7 @@ DynamicValue::Reader DynamicList::Reader::operator[](uint index) {
 
       case schema::Type::Body::LIST_TYPE:
         FAIL_CHECK("elementType should not be LIST_TYPE when depth == 0.");
+        return DynamicValue::Reader();
 
       case schema::Type::Body::STRUCT_TYPE:
         return DynamicValue::Reader(DynamicStruct::Reader(

c++/src/capnproto/dynamic.h

@@ -66,6 +66,14 @@ struct DynamicList {
   class Builder;
 };
 
+template <Kind k> struct DynamicTypeFor_;
+template <> struct DynamicTypeFor_<Kind::ENUM> { typedef DynamicEnum Type; };
+template <> struct DynamicTypeFor_<Kind::STRUCT> { typedef DynamicStruct Type; };
+template <> struct DynamicTypeFor_<Kind::LIST> { typedef DynamicList Type; };
+
+template <typename T>
+using DynamicTypeFor = typename DynamicTypeFor_<kind<T>()>::Type;
+
 class SchemaPool {
   // Class representing a pool of schema data which is indexed for convenient traversal.
   //
@@ -74,21 +82,9 @@ class SchemaPool {
   //   This will make it easier to deal with ownership and to batch-add related nodes.
 
 public:
-  SchemaPool(): base(nullptr), impl(nullptr) {}
-  // Constructs an empty pool.
-
-  explicit SchemaPool(const SchemaPool& base): base(&base), impl(nullptr) {}
-  // Constructs a pool that extends the given base pool.  The behavior is similar to copying the
-  // base pool, except that memory is shared.  The base pool must outlive the new pool.
-  //
-  // The purpose of this is to allow a pool to be shared by multiple threads.  The from*() methods
-  // may add new schema nodes to the pool if they aren't already present, and therefore they are
-  // not thread-safe.  However, if you create a shared pool that contains all the types you need,
-  // then it is reasonably efficient to create a fresh SchemaPool on the stack extending the shared
-  // pool each time you need to manipulate a type dynamically.  If the shared pool ends up
-  // containing everything that is needed, then the extension pool won't allocate any memory at all.
-
+  SchemaPool();
   ~SchemaPool();
+  CAPNPROTO_DISALLOW_COPY(SchemaPool);
 
   void add(schema::Node::Reader node);
   // Add a schema node.  It will be copied and validated, throwing an exception if invalid.  If
@@ -97,20 +93,15 @@ public:
   // an exception will be thrown.
 
   template <typename T>
-  void add();
+  inline void add() { add(internal::rawSchema<T>()); }
   // Add schema for the given compiled-in type and all of its transitive dependencies, including
   // nested nodes, but NOT necessarily including annotation definitions (because those are not
   // always compiled in) or parent scopes (because adding parent scopes would necessarily mean
   // adding all types in the file and in all transitive imports, which may be much more than you
   // want).
 
-  bool has(uint64_t typeId) const;
-
-  template <typename T> DynamicEnum fromEnum(T&& value);
-  template <typename T> DynamicStruct::Reader fromStructReader(T&& reader);
-  template <typename T> DynamicStruct::Builder fromStructBuilder(T&& builder);
-  template <typename T> DynamicList::Reader fromListReader(T&& reader);
-  template <typename T> DynamicList::Builder fromListBuilder(T&& builder);
+  Maybe<schema::Node::Reader> tryGetNode(uint64_t id) const;
+  // Try to look up the node, but return nullptr if it's unknown.
 
   schema::Node::Reader getNode(uint64_t id) const;
   // Look up the node with the given ID, throwing an exception if not found.
@@ -120,16 +111,32 @@ public:
   schema::Node::Reader getInterface(uint64_t id) const;
   // Like getNode() but also throws if the kind is not as requested.
 
+  template <typename T>
+  ReaderFor<DynamicTypeFor<FromReader<T>>> toDynamic(T&& value) const;
+  template <typename T>
+  BuilderFor<DynamicTypeFor<FromBuilder<T>>> toDynamic(T&& value) const;
+  // Convert an arbitrary struct or list reader or builder type into the equivalent dynamic type.
+  // Example:
+  //     DynamicStruct::Reader foo = pool.toDynamic(myType.getFoo());
+
+  template <typename T> DynamicEnum fromEnum(T&& value) const;
+  template <typename T> DynamicStruct::Reader fromStructReader(T&& reader) const;
+  template <typename T> DynamicStruct::Builder fromStructBuilder(T&& builder) const;
+  template <typename T> DynamicList::Reader fromListReader(T&& reader) const;
+  template <typename T> DynamicList::Builder fromListBuilder(T&& builder) const;
+  // Convert native types to dynamic types.
+
 private:
   struct Impl;
 
-  const SchemaPool* base;
   Impl* impl;
 
-  SchemaPool& operator=(const SchemaPool&) = delete;
-
+  void add(const internal::RawSchema& rawSchema);
   void addNoCopy(schema::Node::Reader node);
 
+  template <typename T, Kind k = kind<T>()>
+  struct ToDynamicImpl;
+
   friend class DynamicEnum;
   friend struct DynamicStruct;
   friend struct DynamicList;
@@ -265,8 +272,8 @@ public:
   inline typename T::Reader as() { return asImpl<T>::apply(*this); }
   // Convert the object to the given struct, list, or blob type.
 
-  DynamicStruct::Reader toStruct(schema::Node::Reader schema);
-  DynamicList::Reader toList(schema::Type::Reader elementType);
+  DynamicStruct::Reader asStruct(schema::Node::Reader schema);
+  DynamicList::Reader asList(schema::Type::Reader elementType);
 
 private:
   const SchemaPool* pool;
@@ -279,8 +286,8 @@ private:
   // Implementation backing the as() method.  Needs to be a struct to allow partial
   // specialization.  Has a method apply() which does the work.
 
-  DynamicStruct::Reader toStruct(uint64_t typeId);
-  DynamicList::Reader toList(internal::ListSchema schema);
+  DynamicStruct::Reader asStruct(uint64_t typeId);
+  DynamicList::Reader asList(internal::ListSchema schema);
 
   friend struct DynamicStruct;
   friend struct DynamicList;
@@ -294,8 +301,8 @@ public:
   inline typename T::Builder as() { return asImpl<T>::apply(*this); }
   // Convert the object to the given struct, list, or blob type.
 
-  DynamicStruct::Builder toStruct(schema::Node::Reader schema);
-  DynamicList::Builder toList(schema::Type::Reader elementType);
+  DynamicStruct::Builder asStruct(schema::Node::Reader schema);
+  DynamicList::Builder asList(schema::Type::Reader elementType);
 
 private:
   const SchemaPool* pool;
@@ -308,8 +315,8 @@ private:
   // Implementation backing the as() method.  Needs to be a struct to allow partial
   // specialization.  Has a method apply() which does the work.
 
-  DynamicStruct::Builder toStruct(uint64_t typeId);
-  DynamicList::Builder toList(internal::ListSchema schema);
+  DynamicStruct::Builder asStruct(uint64_t typeId);
+  DynamicList::Builder asList(internal::ListSchema schema);
 
   friend struct DynamicStruct;
   friend struct DynamicList;
@@ -413,6 +420,7 @@ private:
   friend struct DynamicList;
   friend class MessageReader;
   friend class MessageBuilder;
+  friend class SchemaPool;
 };
 
 class DynamicStruct::Builder {
@@ -491,6 +499,7 @@ private:
   friend struct DynamicList;
   friend class MessageReader;
   friend class MessageBuilder;
+  friend class SchemaPool;
 };
 
 // -------------------------------------------------------------------
@@ -542,6 +551,7 @@ private:
   friend struct DynamicStruct;
   friend struct DynamicObject;
   friend class DynamicList::Builder;
+  friend class SchemaPool;
 };
 
 class DynamicList::Builder {
@@ -587,6 +597,7 @@ private:
   friend struct internal::PointerHelpers;
   friend struct DynamicStruct;
   friend struct DynamicObject;
+  friend class SchemaPool;
 };
 
 // -------------------------------------------------------------------
@@ -791,6 +802,35 @@ struct PointerHelpers<DynamicList, Kind::UNKNOWN> {
 // =======================================================================================
 // Inline implementation details.
 
+template <typename T>
+struct SchemaPool::ToDynamicImpl<T, Kind::STRUCT> {
+  static inline DynamicStruct::Reader apply(const SchemaPool* pool, typename T::Reader value) {
+    return DynamicStruct::Reader(pool, pool->getStruct(typeId<T>()), value._reader);
+  }
+  static inline DynamicStruct::Builder apply(const SchemaPool* pool, typename T::Builder value) {
+    return DynamicStruct::Builder(pool, pool->getStruct(typeId<T>()), value._builder);
+  }
+};
+
+template <typename T>
+struct SchemaPool::ToDynamicImpl<T, Kind::LIST> {
+  static inline DynamicList::Reader apply(const SchemaPool* pool, typename T::Reader value) {
+    return DynamicList::Reader(pool, internal::ListSchemaFor<T>::schema, value.reader);
+  }
+  static inline DynamicList::Builder apply(const SchemaPool* pool, typename T::Builder value) {
+    return DynamicList::Builder(pool, internal::ListSchemaFor<T>::schema, value.builder);
+  }
+};
+
+template <typename T>
+ReaderFor<DynamicTypeFor<FromReader<T>>> SchemaPool::toDynamic(T&& value) const {
+  return ToDynamicImpl<FromReader<T>>::apply(this, value);
+}
+template <typename T>
+BuilderFor<DynamicTypeFor<FromBuilder<T>>> SchemaPool::toDynamic(T&& value) const {
+  return ToDynamicImpl<FromBuilder<T>>::apply(this, value);
+}
+
 #define CAPNPROTO_DECLARE_TYPE(name, discrim, typeName) \
 inline DynamicValue::Reader::Reader(ReaderFor<typeName> name##Value) \
     : type(schema::Type::Body::discrim##_TYPE), name##Value(name##Value) {} \
@@ -885,26 +925,26 @@ struct DynamicValue::Builder::asImpl<T, Kind::LIST> {
 template <typename T>
 struct DynamicObject::Reader::asImpl<T, Kind::STRUCT> {
   static T apply(Reader reader) {
-    return reader.toStruct(typeId<T>()).as<T>();
+    return reader.asStruct(typeId<T>()).as<T>();
   }
 };
 template <typename T>
 struct DynamicObject::Builder::asImpl<T, Kind::STRUCT> {
   static T apply(Builder builder) {
-    return builder.toStruct(typeId<T>()).as<T>();
+    return builder.asStruct(typeId<T>()).as<T>();
   }
 };
 
 template <typename T>
 struct DynamicObject::Reader::asImpl<List<T>, Kind::LIST> {
   static T apply(Reader reader) {
-    return reader.toList(internal::ListSchemaForElement<T>::schema).as<T>();
+    return reader.asList(internal::ListSchemaForElement<T>::schema).as<T>();
   }
 };
 template <typename T>
 struct DynamicObject::Builder::asImpl<List<T>, Kind::LIST> {
   static T apply(Builder builder) {
-    return builder.toList(internal::ListSchemaForElement<T>::schema).as<T>();
+    return builder.asList(internal::ListSchemaForElement<T>::schema).as<T>();
   }
 };
 

c++/src/capnproto/generated-header-support.h

@@ -35,11 +35,6 @@ class SchemaPool;   // Needs to be declared for dynamic Object accessors.
 
 class DynamicStruct;  // So that it can be declared a friend.
 
-template <typename T>
-inline constexpr uint64_t typeId();
-// typeId<MyType>() returns the type ID as defined in the schema.  Works with structs, enums, and
-// interfaces.
-
 namespace internal {
 
 template <typename T>
@@ -127,24 +122,50 @@ struct PointerHelpers<TrustedMessage> {
 
 struct RawSchema {
   const word* encodedNode;
-  const RawSchema* dependencies;
+  const RawSchema* const* dependencies;
 };
 
 template <typename T>
-inline constexpr const RawSchema& rawSchema();
+struct RawSchemaFor;
+
+template <typename T>
+inline const RawSchema& rawSchema() {
+  return RawSchemaFor<T>::getSchema();
+}
+
+template <typename T>
+struct TypeIdFor;
 
 }  // namespace internal
+
+template <typename T>
+inline constexpr uint64_t typeId() { return internal::TypeIdFor<T>::typeId; }
+// typeId<MyType>() returns the type ID as defined in the schema.  Works with structs, enums, and
+// interfaces.
+
 }  // namespace capnproto
 
 #define CAPNPROTO_DECLARE_ENUM(type, id) \
-    template <> struct KindOf<type> { static constexpr Kind kind = Kind::ENUM; }
+    template <> struct KindOf<type> { static constexpr Kind kind = Kind::ENUM; }; \
+    template <> struct TypeIdFor<type> { static constexpr uint64_t typeId = 0x##id; }; \
+    template <> struct RawSchemaFor<type> { \
+      static inline const RawSchema& getSchema() { return schemas::s_##id; } \
+    }
 #define CAPNPROTO_DECLARE_STRUCT(type, id, dataWordSize, pointerCount, preferredElementEncoding) \
     template <> struct KindOf<type> { static constexpr Kind kind = Kind::STRUCT; }; \
     template <> struct StructSizeFor<type> { \
       static constexpr StructSize value = StructSize( \
           dataWordSize * WORDS, pointerCount * REFERENCES, FieldSize::preferredElementEncoding); \
+    }; \
+    template <> struct TypeIdFor<type> { static constexpr uint64_t typeId = 0x##id; }; \
+    template <> struct RawSchemaFor<type> { \
+      static inline const RawSchema& getSchema() { return schemas::s_##id; } \
     }
 #define CAPNPROTO_DECLARE_INTERFACE(type, id) \
-    template <> struct KindOf<type> { static constexpr Kind kind = Kind::INTERFACE; }
+    template <> struct KindOf<type> { static constexpr Kind kind = Kind::INTERFACE; }; \
+    template <> struct TypeIdFor<type> { static constexpr uint64_t typeId = 0x##id; }; \
+    template <> struct RawSchemaFor<type> { \
+      static inline const RawSchema& getSchema() { return schemas::s_##id; } \
+    }
 
 #endif  // CAPNPROTO_GENERATED_HEADER_SUPPORT_H_

c++/src/capnproto/list.h

@@ -62,6 +62,14 @@ template <typename T>
 using BuilderFor = typename internal::MaybeReaderBuilder<T>::Builder;
 // The type returned by List<T>::Builder::operator[].
 
+template <typename T>
+using FromReader = typename RemoveReference<T>::Reads;
+// FromReader<MyType::Reader> = MyType (for any Cap'n Proto type).
+
+template <typename T>
+using FromBuilder = typename RemoveReference<T>::Builds;
+// FromBuilder<MyType::Builder> = MyType (for any Cap'n Proto type).
+
 namespace internal {
 
 template <typename T, Kind k> struct KindOf<List<T, k>> { static constexpr Kind kind = Kind::LIST; };
@@ -169,6 +177,8 @@ struct List<T, Kind::PRIMITIVE> {
 
   class Reader {
   public:
+    typedef List<T> Reads;
+
     Reader() = default;
     inline explicit Reader(internal::ListReader reader): reader(reader) {}
 
@@ -187,6 +197,8 @@ struct List<T, Kind::PRIMITIVE> {
 
   class Builder {
   public:
+    typedef List<T> Builds;
+
     Builder() = default;
     inline explicit Builder(internal::ListBuilder builder): builder(builder) {}
 
@@ -274,6 +286,8 @@ struct List<T, Kind::STRUCT> {
 
   class Reader {
   public:
+    typedef List<T> Reads;
+
     Reader() = default;
     inline explicit Reader(internal::ListReader reader): reader(reader) {}
 
@@ -292,6 +306,8 @@ struct List<T, Kind::STRUCT> {
 
   class Builder {
   public:
+    typedef List<T> Builds;
+
     Builder() = default;
     inline explicit Builder(internal::ListBuilder builder): builder(builder) {}
 
@@ -353,6 +369,8 @@ struct List<List<T>, Kind::LIST> {
 
   class Reader {
   public:
+    typedef List<List<T>> Reads;
+
     Reader() = default;
     inline explicit Reader(internal::ListReader reader): reader(reader) {}
 
@@ -371,6 +389,8 @@ struct List<List<T>, Kind::LIST> {
 
   class Builder {
   public:
+    typedef List<List<T>> Builds;
+
     Builder() = default;
     inline explicit Builder(internal::ListBuilder builder): builder(builder) {}
 
@@ -433,6 +453,8 @@ template <typename T>
 struct List<T, Kind::BLOB> {
   class Reader {
   public:
+    typedef List<T> Reads;
+
     Reader() = default;
     inline explicit Reader(internal::ListReader reader): reader(reader) {}
 
@@ -451,6 +473,8 @@ struct List<T, Kind::BLOB> {
 
   class Builder {
   public:
+    typedef List<T> Builds;
+
     Builder() = default;
     inline explicit Builder(internal::ListBuilder builder): builder(builder) {}
 

c++/src/capnproto/type-safety.h

@@ -52,21 +52,24 @@ struct NoInfer {
   typedef T Type;
 };
 
-template <typename T> struct RemoveReference { typedef T Type; };
-template <typename T> struct RemoveReference<T&> { typedef T Type; };
+template <typename T> struct RemoveReference_ { typedef T Type; };
+template <typename T> struct RemoveReference_<T&> { typedef T Type; };
 template <typename> struct IsLvalueReference { static constexpr bool value = false; };
 template <typename T> struct IsLvalueReference<T&> { static constexpr bool value = true; };
 
+template <typename T>
+using RemoveReference = typename RemoveReference_<T>::Type;
+
 // #including <utility> just for std::move() and std::forward() is excessive.  Instead, we
 // re-define them here.
 
 template<typename T> constexpr T&& move(T& t) noexcept { return static_cast<T&&>(t); }
 
 template<typename T>
-constexpr T&& forward(typename RemoveReference<T>::Type& t) noexcept {
+constexpr T&& forward(RemoveReference<T>& t) noexcept {
   return static_cast<T&&>(t);
 }
-template<typename T> constexpr T&& forward(typename RemoveReference<T>::Type&& t) noexcept {
+template<typename T> constexpr T&& forward(RemoveReference<T>&& t) noexcept {
   static_assert(!IsLvalueReference<T>::value, "Attempting to forward rvalue as lvalue reference.");
   return static_cast<T&&>(t);
 }

compiler/src/Compiler.hs

@@ -40,6 +40,7 @@ import Text.Printf(printf)
 import qualified Data.Digest.MD5 as MD5
 import qualified Codec.Binary.UTF8.String as UTF8
 import Util(delimit, intToBytes)
+import Data.Bits(setBit)
 
 ------------------------------------------------------------------------------------------
 -- Error helpers
@@ -412,7 +413,7 @@ childId name Nothing parent = let
     hash = MD5.hash (intToBytes (descId parent) 8 ++ UTF8.encode name)
     addByte :: Word64 -> Word8 -> Word64
     addByte b v = b * 256 + fromIntegral v
-    in foldl addByte 0 (take 8 hash)
+    in flip setBit 63 $ foldl addByte 0 (take 8 hash)
 
 ------------------------------------------------------------------------------------------
 

compiler/src/Main.hs

@@ -48,12 +48,13 @@ import qualified Data.ByteString.Lazy.Char8 as LZ
 import Data.ByteString(unpack, pack, hPut)
 import Data.Word(Word64, Word8)
 import Data.Maybe(fromMaybe, catMaybes)
+import Data.Function(on)
 import Semantics
 import WireFormat(encodeSchema)
 
 import CxxGenerator(generateCxx)
 
-type GeneratorFn = [FileDesc] -> [FileDesc] -> IO [(FilePath, LZ.ByteString)]
+type GeneratorFn = [FileDesc] -> [Word8] -> Map.Map Word64 [Word8] -> IO [(FilePath, LZ.ByteString)]
 
 generatorFns :: Map.Map String GeneratorFn
 generatorFns = Map.fromList [ ("c++", generateCxx) ]
@@ -120,7 +121,24 @@ main = do
         evalStateT (handleFiles isVerbose searchPath files)
                    (CompilerState False Map.empty)
 
-    mapM_ (doOutput requestedFiles allFiles) outputs
+    let (schema, schemaNodes) = encodeSchema requestedFiles allFiles
+        toEntry ((i, _), node) = (i, node)
+        schemaMap = Map.fromList $ map toEntry schemaNodes
+        areDupes (i, _) (j, _) = i == j
+        dupes = filter (\x -> length x > 1) $ List.groupBy areDupes
+              $ List.sortBy (compare `on` fst) $ map fst schemaNodes
+
+    unless (null dupes) (do
+        hPutStr stderr $ concat
+            ("Duplicate type / delcaration IDs detected:\n":
+             map (concatMap (uncurry $ printf "  @0x%016x %s\n")) dupes)
+        hPutStr stderr
+            "IDs (16-digit hex strings prefixed with @0x) must be unique.  Sorry I'm not\n\
+            \able to be more specific about where the duplicates were seen, but it should\n\
+            \be easy enough to grep, right?"
+        exitFailure)
+
+    mapM_ (doOutput requestedFiles schema schemaMap) outputs
 
     when failed exitFailure
 
@@ -140,8 +158,7 @@ parseOutputArg str = let
 
 pluginName lang = if '/' `elem` lang then lang else "capnpc-" ++ lang
 
-callPlugin lang wd descs transitiveImports = do
-    let schema = encodeSchema descs transitiveImports
+callPlugin lang wd _ schema _ = do
     (Just hin, _, _, p) <- createProcess (proc (pluginName lang) [])
         { std_in = CreatePipe, cwd = wd }
     hSetBinaryMode hin True
@@ -271,13 +288,13 @@ handleFile isVerbose searchPath filename = do
         Right _ -> return Nothing
         Left desc -> return $ Just desc
 
-doOutput requestedFiles allFiles output = do
+doOutput requestedFiles schema schemaMap output = do
     let write dir (name, content) = do
             let outFilename = dir ++ "/" ++ name
             createDirectoryIfMissing True $ takeDirectory outFilename
             LZ.writeFile outFilename content
         generate (generatorFn, dir) = do
-            files <- generatorFn requestedFiles allFiles
+            files <- generatorFn requestedFiles schema schemaMap
             mapM_ (write dir) files
     liftIO $ generate output
 

compiler/src/WireFormat.hs

@@ -350,8 +350,12 @@ encodeMessage _ _ = error "Not a message."
 
 type EncodedPtr = (Integer -> [Word8], [Word8])
 
-encodeSchema :: [FileDesc] -> [FileDesc] -> [Word8]
-encodeSchema requestedFiles allFiles = encRoot where
+-- Given the list of requested files and the list of all files including transitive imports,
+-- returns a tuple containing the appropriate encoded CodeGeneratorRequest as well as a list
+-- of ((typeId, displayName), encodedNode), where encodedNode is the encoded schema node
+-- appropriate for reading as a "trusted message".
+encodeSchema :: [FileDesc] -> [FileDesc] -> ([Word8], [((Word64, String), [Word8])])
+encodeSchema requestedFiles allFiles = (encRoot, nodesForEmbedding) where
     encUInt64 = EncodedBytes . flip intToBytes 8
     encUInt32 = EncodedBytes . flip intToBytes 4
     encUInt16 :: (Integral a, Bits a) => a -> EncodedData
@@ -366,10 +370,11 @@ encodeSchema requestedFiles allFiles = encRoot where
               $ zip [0,elemBits..] elements
         in (encodeListReference (SizeData elementSize) (genericLength elements), bytes)
 
-    encPtrList :: [EncodedPtr] -> EncodedPtr
-    encPtrList elements = let
-        (ptrBytes, childBytes) = packPointers (genericLength elements) (zip [0..] elements) 0
-        in (encodeListReference SizeReference (genericLength elements), ptrBytes ++ childBytes)
+    -- Not used, but maybe useful in the future.
+    --encPtrList :: [EncodedPtr] -> EncodedPtr
+    --encPtrList elements = let
+    --    (ptrBytes, childBytes) = packPointers (genericLength elements) (zip [0..] elements) 0
+    --    in (encodeListReference SizeReference (genericLength elements), ptrBytes ++ childBytes)
 
     encStructList :: (DataSectionSize, Integer)
                   -> [([(Integer, EncodedData)], [(Integer, EncodedPtr)])]
@@ -429,6 +434,10 @@ encodeSchema requestedFiles allFiles = encRoot where
 
     allDescs = concatMap flattenDescs $ map DescFile allFiles
 
+    allNodes = map encNode allDescs
+
+    nodesForEmbedding = map encodeNodeForEmbedding allNodes
+
     ---------------------------------------------
 
     encRoot = let
@@ -437,10 +446,15 @@ encodeSchema requestedFiles allFiles = encRoot where
         segment = ptrBytes ++ childBytes
         in concat [[0,0,0,0], intToBytes (div (length segment) 8) 4, segment]
 
+    encodeNodeForEmbedding ((typeId, name), node) = let
+        ptrVal = encStruct nodeSize node
+        (ptrBytes, childBytes) = packPointers 1 [(0, ptrVal)] 0
+        in ((typeId, name), ptrBytes ++ childBytes)
+
     codeGeneratorRequestSize = (DataSectionWords 0, 2)
     encCodeGeneratorRequest = (dataValues, ptrValues) where
         dataValues = []
-        ptrValues = [ (0, encStructList nodeSize $ map encNode allDescs)
+        ptrValues = [ (0, encStructList nodeSize $ map snd allNodes)
                     , (1, encDataList Size64 $ map (encUInt64 . fileId) requestedFiles)
                     ]
 
@@ -527,18 +541,20 @@ encodeSchema requestedFiles allFiles = encRoot where
         encStructList annotationSize $ map encAnnotation $ Map.toList annotations
 
     nodeSize = (DataSectionWords 3, 4)
-    encNode :: Desc -> ([(Integer, EncodedData)], [(Integer, EncodedPtr)])
-    encNode desc = (dataValues, ptrValues) where
+    encNode :: Desc -> ((Word64, String), ([(Integer, EncodedData)], [(Integer, EncodedPtr)]))
+    encNode desc = ((descId desc, dname), (dataValues, ptrValues)) where
         dataValues = [ (0, encUInt64 $ descId desc)
                      , (64, encUInt64 $ scopedId desc)
                      , (128, encUInt16 discrim)
                      ]
-        ptrValues = [ (0, encText $ displayName desc)
+        ptrValues = [ (0, encText dname)
                     , (1, encStructList nestedNodeSize $ map encNestedNode $ descNestedNodes desc)
                     , (2, encAnnotationList $ descAnnotations desc)
                     , (3, encStruct bodySize body)
                     ]
 
+        dname = displayName desc
+
         (discrim, bodySize, body) = case desc of
             DescFile d -> (0::Word16, fileNodeSize, encFileNode d)
             DescStruct d -> (1, structNodeSize, encStructNode d)
@@ -604,7 +620,7 @@ encodeSchema requestedFiles allFiles = encRoot where
                           ]
             ptrValues2 = [ (0, encText $ fieldName field)
                          , (1, encAnnotationList $ fieldAnnotations field)
-                         , (2, encStruct (DataSection32, 2) (dataValues3, ptrValues3))
+                         , (2, encStruct (DataSectionWords 1, 2) (dataValues3, ptrValues3))
                          ]
 
             -- StructNode.Field

compiler/src/c++-header.mustache

@@ -100,6 +100,13 @@ enum class {{enumName}}: uint16_t {
 {{/fileNamespaces}}
 
 namespace capnproto {
+namespace schemas {
+{{#fileTypes}}
+{{#typeSchema}}
+extern const ::capnproto::internal::RawSchema s_{{schemaId}};
+{{/typeSchema}}
+{{/fileTypes}}
+}  // namespace schemas
 namespace internal {
 {{#fileTypes}}
 {{#typeStructOrUnion}}
@@ -131,6 +138,8 @@ namespace {{namespaceName}} {
 
 class {{typeFullName}}::Reader {
 public:
+  typedef {{typeName}} Reads;
+
   Reader() = default;
   inline explicit Reader(::capnproto::internal::StructReader base): _reader(base) {}
 {{#typeStruct}}
@@ -163,11 +172,14 @@ public:
 {{/typeFields}}
 private:
   ::capnproto::internal::StructReader _reader;
+  friend class ::capnproto::SchemaPool;
 };
 {{! ------------------------------------------------------------------------------------------- }}
 
 class {{typeFullName}}::Builder {
 public:
+  typedef {{typeName}} Builds;
+
   Builder() = default;
   inline explicit Builder(::capnproto::internal::StructBuilder base): _builder(base) {}
   inline operator Reader() { return Reader(_builder.asReader()); }
@@ -220,6 +232,7 @@ public:
 {{/typeFields}}
 private:
   ::capnproto::internal::StructBuilder _builder;
+  friend class ::capnproto::SchemaPool;
 };
 {{/typeStructOrUnion}}
 {{/fileTypes}}

compiler/src/c++-source.mustache

@@ -49,3 +49,26 @@ const ::capnproto::internal::AlignedData<{{defaultWordCount}}>
 {{#fileNamespaces}}
 }  // namespace
 {{/fileNamespaces}}
+
+namespace capnproto {
+namespace schemas {
+
+{{#fileTypes}}
+{{#typeSchema}}
+static const ::capnproto::internal::AlignedData<{{schemaWordCount}}> b_{{schemaId}} = {
+  { {{schemaBytes}} }
+};
+static const ::capnproto::internal::RawSchema* const d_{{schemaId}}[] = {
+{{#schemaDependencies}}
+  &s_{{dependencyId}},
+{{/schemaDependencies}}
+  nullptr
+};
+const ::capnproto::internal::RawSchema s_{{schemaId}} = {
+  b_{{schemaId}}.words, d_{{schemaId}}
+};
+{{/typeSchema}}
+{{/fileTypes}}
+
+}  // namespace schemas
+}  // namespace capnproto