C++ code generator WIP.

c++/src/capnp/compiler/capnp.c++

@@ -43,8 +43,10 @@
 
 #if HAVE_CONFIG_H
 #include "config.h"
-#else
-#define VERSION "(unknown ekam build)"
+#endif
+
+#ifndef VERSION
+#define VERSION "(unknown)"
 #endif
 
 namespace capnp {

c++/src/capnp/compiler/capnpc-capnp.c++

@@ -38,8 +38,10 @@
 
 #if HAVE_CONFIG_H
 #include "config.h"
-#else
-#define VERSION "(unknown ekam build)"
+#endif
+
+#ifndef VERSION
+#define VERSION "(unknown)"
 #endif
 
 namespace capnp {

c++/src/capnp/generated-header-support.h

@@ -153,6 +153,9 @@ struct RawSchema {
   const word* encodedNode;
   // Encoded SchemaNode, readable via readMessageUnchecked<schema::Node>(encodedNode).
 
+  uint32_t encodedSize;
+  // Size of encodedNode, in words.
+
   const RawSchema* const* dependencies;
   // Pointers to other types on which this one depends, sorted by ID.  The schemas in this table
   // may be uninitialized -- you must call ensureInitialized() on the one you wish to use before

c++/src/capnp/schema-loader.c++

@@ -1032,6 +1032,7 @@ _::RawSchema* SchemaLoader::Impl::load(const schema::Node::Reader& reader, bool
   if (shouldReplace) {
     // Initialize the RawSchema.
     slot->encodedNode = validated.begin();
+    slot->encodedSize = validated.size();
     slot->dependencies = validator.makeDependencyArray(&slot->dependencyCount);
     slot->membersByName = validator.makeMemberInfoArray(&slot->memberCount);
   }

c++/src/capnp/schema.c++

@@ -31,6 +31,10 @@ schema::Node::Reader Schema::getProto() const {
   return readMessageUnchecked<schema::Node>(raw->encodedNode);
 }
 
+kj::ArrayPtr<const word> Schema::asUncheckedMessage() const {
+  return kj::arrayPtr(raw->encodedNode, raw->encodedSize);
+}
+
 Schema Schema::getDependency(uint64_t id) const {
   uint lower = 0;
   uint upper = raw->dependencyCount;
@@ -141,6 +145,13 @@ kj::Maybe<StructSchema::Union> StructSchema::Member::getContainingUnion() const
   return parent.getMembers()[unionIndex - 1].asUnion();
 }
 
+StructSchema::Field StructSchema::Member::asField() const {
+  KJ_REQUIRE(proto.getBody().which() == schema::StructNode::Member::Body::FIELD_MEMBER,
+          "Tried to use non-field struct member as a field.",
+          parent.getProto().getDisplayName(), proto.getName());
+  return Field(*this);
+}
+
 StructSchema::Union StructSchema::Member::asUnion() const {
   KJ_REQUIRE(proto.getBody().which() == schema::StructNode::Member::Body::UNION_MEMBER,
           "Tried to use non-union struct member as a union.",
@@ -148,6 +159,41 @@ StructSchema::Union StructSchema::Member::asUnion() const {
   return Union(*this);
 }
 
+StructSchema::Group StructSchema::Member::asGroup() const {
+  KJ_REQUIRE(proto.getBody().which() == schema::StructNode::Member::Body::GROUP_MEMBER,
+          "Tried to use non-group struct member as a group.",
+          parent.getProto().getDisplayName(), proto.getName());
+  return Group(*this);
+}
+
+uint32_t StructSchema::Field::getDefaultValueSchemaOffset() const {
+  auto defaultValue = proto.getBody().getFieldMember().getDefaultValue().getBody();
+  const word* ptr;
+
+  switch (defaultValue.which()) {
+    case schema::Value::Body::TEXT_VALUE:
+      ptr = reinterpret_cast<const word*>(defaultValue.getTextValue().begin());
+      break;
+    case schema::Value::Body::DATA_VALUE:
+      ptr = reinterpret_cast<const word*>(defaultValue.getDataValue().begin());
+      break;
+    case schema::Value::Body::STRUCT_VALUE:
+      ptr = defaultValue.getStructValue<_::UncheckedMessage>();
+      break;
+    case schema::Value::Body::LIST_VALUE:
+      ptr = defaultValue.getListValue<_::UncheckedMessage>();
+      break;
+    case schema::Value::Body::OBJECT_VALUE:
+      ptr = defaultValue.getObjectValue<_::UncheckedMessage>();
+      break;
+    default:
+      KJ_FAIL_ASSERT("getDefaultValueSchemaOffset() can only be called on struct, list, "
+                     "and object fields.");
+  }
+
+  return ptr - parent.raw->encodedNode;
+}
+
 StructSchema::MemberList StructSchema::Union::getMembers() const {
   return MemberList(parent, index + 1, proto.getBody().getUnionMember().getMembers());
 }
@@ -164,6 +210,25 @@ StructSchema::Member StructSchema::Union::getMemberByName(kj::StringPtr name) co
   }
 }
 
+StructSchema::MemberList StructSchema::Group::getMembers() const {
+  return MemberList(parent, 0, proto.getBody().getGroupMember().getMembers());
+}
+
+#if 0
+// TODO(soon):  Implement correctly.  Requires some changes to lookup table format.
+kj::Maybe<StructSchema::Member> StructSchema::Group::findMemberByName(kj::StringPtr name) const {
+  return findSchemaMemberByName(parent.raw, name, index + 1, getMembers());
+}
+
+StructSchema::Member StructSchema::Group::getMemberByName(kj::StringPtr name) const {
+  KJ_IF_MAYBE(member, findMemberByName(name)) {
+    return *member;
+  } else {
+    KJ_FAIL_REQUIRE("group has no such member", name);
+  }
+}
+#endif
+
 // -------------------------------------------------------------------
 
 EnumSchema::EnumerantList EnumSchema::getEnumerants() const {

c++/src/capnp/schema.h

@@ -58,6 +58,10 @@ public:
 
   schema::Node::Reader getProto() const;
 
+  kj::ArrayPtr<const word> asUncheckedMessage() const;
+  // Get the encoded schema node content as a single message segment.  It is safe to read as an
+  // unchecked message.
+
   Schema getDependency(uint64_t id) const;
   // Gets the Schema for one of this Schema's dependencies.  For example, if this Schema is for a
   // struct, you could look up the schema for one of its fields' types.  Throws an exception if this
@@ -111,7 +115,9 @@ public:
   StructSchema() = default;
 
   class Member;
+  class Field;
   class Union;
+  class Group;
   class MemberList;
 
   MemberList getMembers() const;
@@ -146,9 +152,15 @@ public:
   kj::Maybe<Union> getContainingUnion() const;
   // If this a member of a union, gets the containing union schema.
 
+  Field asField() const;
+  // Cast the member to a Field.  Throws an exception if not a field.
+
   Union asUnion() const;
   // Cast the member to a Union.  Throws an exception if not a union.
 
+  Group asGroup() const;
+  // Cast the member to a Group.  Throws an exception if not a group.
+
   inline bool operator==(const Member& other) const;
   inline bool operator!=(const Member& other) const { return !(*this == other); }
 
@@ -156,8 +168,7 @@ private:
   StructSchema parent;
   uint unionIndex;  // 0 = none, >0 = actual union index - 1
   uint index;
-  mutable schema::StructNode::Member::Reader proto;
-  // TODO(soon): Make all reader methods const and then remove this ugly use of "mutable".
+  schema::StructNode::Member::Reader proto;
 
   inline Member(StructSchema parent, uint unionIndex, uint index,
                 schema::StructNode::Member::Reader proto)
@@ -166,6 +177,36 @@ private:
   friend class StructSchema;
 };
 
+class StructSchema::Field: public Member {
+public:
+  Field() = default;
+
+  uint32_t getDefaultValueSchemaOffset() const;
+  // For struct, list, and object fields, returns the offset, in words, within the first segment of
+  // the struct's schema, where this field's default value pointer is located.  The schema is
+  // always stored as a single-segment unchecked message, which in turn means that the default
+  // value pointer itself can be treated as the root of an unchecked message -- if you know where
+  // to find it, which is what this method helps you with.
+  //
+  // For blobs, returns the offset of the begging of the blob's content within the first segment of
+  // the struct's schema.
+  //
+  // This is primarily useful for code generators.  The C++ code generator, for example, embeds
+  // the entire schema as a raw word array within the generated code.  Of course, to implement
+  // field accessors, it needs access to those fields' default values.  Embedding separate copies
+  // of those default values would be redundant since they are already included in the schema, but
+  // seeking through the schema at runtime to find the default values would be ugly.  Instead,
+  // the code generator can use getDefaultValueSchemaOffset() to find the offset of the default
+  // value within the schema, and can simply apply that offset at runtime.
+  //
+  // If the above does not make sense, you probably don't need this method.
+
+private:
+  inline Field(const Member& base): Member(base) {}
+
+  friend class StructSchema;
+};
+
 class StructSchema::Union: public Member {
 public:
   Union() = default;
@@ -183,6 +224,23 @@ private:
   friend class StructSchema;
 };
 
+class StructSchema::Group: public Member {
+public:
+  Group() = default;
+
+  MemberList getMembers() const;
+
+  kj::Maybe<Member> findMemberByName(kj::StringPtr name) const;
+
+  Member getMemberByName(kj::StringPtr name) const;
+  // Like findMemberByName() but throws an exception on failure.
+
+private:
+  inline Group(const Member& base): Member(base) {}
+
+  friend class StructSchema;
+};
+
 class StructSchema::MemberList {
 public:
   inline uint size() const { return list.size(); }
@@ -233,9 +291,10 @@ public:
   Enumerant() = default;
 
   inline schema::EnumNode::Enumerant::Reader getProto() const { return proto; }
-  inline EnumSchema getContainingEnum() { return parent; }
+  inline EnumSchema getContainingEnum() const { return parent; }
 
-  inline uint16_t getOrdinal() { return ordinal; }
+  inline uint16_t getOrdinal() const { return ordinal; }
+  inline uint getIndex() const { return ordinal; }
 
   inline bool operator==(const Enumerant& other) const;
   inline bool operator!=(const Enumerant& other) const { return !(*this == other); }
@@ -243,8 +302,7 @@ public:
 private:
   EnumSchema parent;
   uint16_t ordinal;
-  mutable schema::EnumNode::Enumerant::Reader proto;
-  // TODO(soon): Make all reader methods const and then remove this ugly use of "mutable".
+  schema::EnumNode::Enumerant::Reader proto;
 
   inline Enumerant(EnumSchema parent, uint16_t ordinal, schema::EnumNode::Enumerant::Reader proto)
       : parent(parent), ordinal(ordinal), proto(proto) {}
@@ -300,9 +358,10 @@ public:
   Method() = default;
 
   inline schema::InterfaceNode::Method::Reader getProto() const { return proto; }
-  inline InterfaceSchema getContainingInterface() { return parent; }
+  inline InterfaceSchema getContainingInterface() const { return parent; }
 
-  inline uint16_t getOrdinal() { return ordinal; }
+  inline uint16_t getOrdinal() const { return ordinal; }
+  inline uint getIndex() const { return ordinal; }
 
   inline bool operator==(const Method& other) const;
   inline bool operator!=(const Method& other) const { return !(*this == other); }
@@ -310,8 +369,7 @@ public:
 private:
   InterfaceSchema parent;
   uint16_t ordinal;
-  mutable schema::InterfaceNode::Method::Reader proto;
-  // TODO(soon): Make all reader methods const and then remove this ugly use of "mutable".
+  schema::InterfaceNode::Method::Reader proto;
 
   inline Method(InterfaceSchema parent, uint16_t ordinal,
                 schema::InterfaceNode::Method::Reader proto)

c++/src/kj/array.h

@@ -469,7 +469,7 @@ private:
 // KJ_MAP_ARRAY
 
 #define KJ_MAP(array, elementName) \
-  ::kj::_::Mapper<decltype(array)>(array) * [&](decltype(*(array).begin()) elementName)
+  ::kj::_::Mapper<KJ_DECLTYPE_REF(array)>(array) * [&](decltype(*(array).begin()) elementName)
 // Applies some function to every element of an array, returning an Array of the results,  with
 // nice syntax.  Example:
 //
@@ -482,7 +482,7 @@ namespace _ {  // private
 template <typename T>
 struct Mapper {
   T array;
-  Mapper(T array): array(kj::fwd<T>(array)) {}
+  Mapper(T&& array): array(kj::fwd<T>(array)) {}
   template <typename Func>
   auto operator*(Func&& func) -> Array<decltype(func(*array.begin()))> {
     auto builder = heapArrayBuilder<decltype(func(*array.begin()))>(array.size());

c++/src/kj/common.h

@@ -293,6 +293,22 @@ template <typename T> struct IsReference_ { static constexpr bool value = false;
 template <typename T> struct IsReference_<T&> { static constexpr bool value = true; };
 template <typename T> constexpr bool isReference() { return IsReference_<T>::value; }
 
+namespace _ {  // private
+
+template <typename T>
+T refIfLvalue(T&&);
+
+}  // namespace _ (private)
+
+#define KJ_DECLTYPE_REF(exp) decltype(::kj::_::refIfLvalue(exp))
+// Like decltype(exp), but if exp is an lvalue, produces a reference type.
+//
+//     int i;
+//     decltype(i) i1(i);                         // i1 has type int.
+//     KJ_DECLTYPE_REF(i + 1) i2(i + 1);          // i2 has type int.
+//     KJ_DECLTYPE_REF(i) i3(i);                  // i3 has type int&.
+//     KJ_DECLTYPE_REF(kj::mv(i)) i4(kj::mv(i));  // i4 has type int.
+
 // =======================================================================================
 // Equivalents to std::move() and std::forward(), since these are very commonly needed and the
 // std header <utility> pulls in lots of other stuff.
@@ -309,6 +325,86 @@ inline constexpr auto min(T&& a, U&& b) -> decltype(a < b ? a : b) { return a <
 template <typename T, typename U>
 inline constexpr auto max(T&& a, U&& b) -> decltype(a > b ? a : b) { return a > b ? a : b; }
 
+// =======================================================================================
+// Useful fake containers
+
+template <typename T>
+class Range {
+public:
+  inline constexpr Range(const T& begin, const T& end): begin_(begin), end_(end) {}
+
+  class Iterator {
+  public:
+    Iterator() = default;
+    inline Iterator(const T& value): value(value) {}
+
+    inline const T& operator*() const { return value; }
+    inline Iterator& operator++() { ++value; return *this; }
+    inline Iterator operator++(int) { return Iterator(value++); }
+    inline bool operator==(const Iterator& other) const { return value == other.value; }
+    inline bool operator!=(const Iterator& other) const { return value != other.value; }
+
+  private:
+    T value;
+  };
+
+  inline Iterator begin() const { return Iterator(begin_); }
+  inline Iterator end() const { return Iterator(end_); }
+
+  inline auto size() const -> decltype(instance<T>() - instance<T>()) { return end_ - begin_; }
+
+private:
+  T begin_;
+  T end_;
+};
+
+template <typename T>
+inline constexpr Range<Decay<T>> range(T&& begin, T&& end) { return Range<Decay<T>>(begin, end); }
+// Returns a fake iterable container containing all values of T from `begin` (inclusive) to `end`
+// (exclusive).  Example:
+//
+//     // Prints 1, 2, 3, 4, 5, 6, 7, 8, 9.
+//     for (int i: kj::range(1, 10)) { print(i); }
+
+template <typename T>
+class Repeat {
+public:
+  inline constexpr Repeat(const T& value, size_t count): value(value), count(count) {}
+
+  class Iterator {
+  public:
+    Iterator() = default;
+    inline Iterator(const T& value, size_t index): value(value), index(index) {}
+
+    inline const T& operator*() const { return value; }
+    inline Iterator& operator++() { ++index; return *this; }
+    inline Iterator operator++(int) { return Iterator(value, index++); }
+    inline bool operator==(const Iterator& other) const { return index == other.index; }
+    inline bool operator!=(const Iterator& other) const { return index != other.index; }
+
+  private:
+    T value;
+    size_t index;
+  };
+
+  inline Iterator begin() const { return Iterator(value, 0); }
+  inline Iterator end() const { return Iterator(value, count); }
+
+  inline size_t size() const { return count; }
+
+private:
+  T value;
+  size_t count;
+};
+
+template <typename T>
+inline constexpr Repeat<Decay<T>> repeat(T&& value, size_t count) {
+  // Returns a fake iterable which contains `count` repeats of `value`.  Useful for e.g. creating
+  // a bunch of spaces:  `kj::repeat(' ', indent * 2)`
+
+  return Repeat<Decay<T>>(value, count);
+}
+
 // =======================================================================================
 // Manually invoking constructors and destructors
 //

c++/src/kj/function.h

@@ -117,19 +117,6 @@ private:
   Own<Iface> impl;
 };
 
-namespace _ {  // private
-
-template <typename T>
-T rvalueOrRef(T&&);
-// Hack to help detect if an expression is an lvalue or an rvalue.
-//
-//     int i;
-//     decltype(i) i1(i);                              // i1 has type int.
-//     decltype(rvalueOrRef(i)) i2(i);                 // i2 has type int&.
-//     decltype(rvalueOrRef(kj::mv(i)) i3(kj::mv(i));  // i3 has type int.
-
-}  // namespace _ (private)
-
 #if 1
 
 namespace _ {  // private
@@ -153,7 +140,7 @@ private:
 }  // namespace _ (private)
 
 #define KJ_BIND_METHOD(obj, method) \
-  ::kj::_::BoundMethod<decltype(::kj::_::rvalueOrRef(obj)), \
+  ::kj::_::BoundMethod<KJ_DECLTYPE_REF(obj), \
                        decltype(&::kj::Decay<decltype(obj)>::method), \
                        &::kj::Decay<decltype(obj)>::method>(obj)
 // Macro that produces a functor object which forwards to the method `obj.name`.  If `obj` is an
@@ -172,7 +159,7 @@ private:
 
 #define KJ_BIND_METHOD(obj, method) \
   ({ \
-    typedef decltype(::kj::_::rvalueOrRef(obj)) T; \
+    typedef KJ_DECLTYPE_REF(obj) T; \
     class F { \
     public: \
       inline F(T&& t): t(::kj::fwd<T>(t)) {} \

c++/src/kj/string.h

@@ -237,6 +237,9 @@ struct Stringifier {
   inline ArrayPtr<const char> operator*(const String& s) const { return s.asArray(); }
   inline ArrayPtr<const char> operator*(const StringPtr& s) const { return s.asArray(); }
 
+  inline Range<char> operator*(const Range<char>& r) const { return r; }
+  inline Repeat<char> operator*(const Repeat<char>& r) const { return r; }
+
   inline FixedArray<char, 1> operator*(char c) const {
     FixedArray<char, 1> result;
     result[0] = c;

compiler/src/CxxGenerator.hs

@@ -75,11 +75,11 @@ globalName desc = globalName (descParent desc) ++ "::" ++ descName desc
 flattenTypes :: [Desc] -> [Desc]
 flattenTypes [] = []
 flattenTypes (d@(DescStruct s):rest) = d:(flattenTypes children ++ flattenTypes rest) where
-    children = catMaybes $ Map.elems $ structMemberMap s
+    children = structMembers s
 flattenTypes (d@(DescUnion u):rest) = d:(flattenTypes children ++ flattenTypes rest) where
-    children = catMaybes $ Map.elems $ unionMemberMap u
+    children = unionMembers u
 flattenTypes (d@(DescInterface i):rest) = d:(flattenTypes children ++ flattenTypes rest) where
-    children = catMaybes $ Map.elems $ interfaceMemberMap i
+    children = interfaceMembers i
 flattenTypes (d@(DescEnum _):rest) = d:flattenTypes rest
 flattenTypes (_:rest) = flattenTypes rest
 
@@ -521,7 +521,7 @@ outerFileContext schemaNodes = fileContext where
         context s = parent s
 
     fileContext desc = mkStrContext context where
-        flattenedMembers = flattenTypes $ catMaybes $ Map.elems $ fileMemberMap desc
+        flattenedMembers = flattenTypes $ fileMembers desc
 
         namespace = maybe [] (splitOn "::") $ fileNamespace desc
 

compiler/src/c++-source.mustache

@@ -69,7 +69,7 @@ static const ::capnp::_::RawSchema::MemberInfo m_{{schemaId}}[] = {
 {{/schemaMembersByName}}
 };
 const ::capnp::_::RawSchema s_{{schemaId}} = {
-  0x{{schemaId}}, b_{{schemaId}}.words, d_{{schemaId}}, m_{{schemaId}},
+  0x{{schemaId}}, b_{{schemaId}}.words, {{schemaWordCount}}, d_{{schemaId}}, m_{{schemaId}},
   {{schemaDependencyCount}}, {{schemaMemberCount}}, nullptr, nullptr
 };
 {{/typeSchema}}