More WIP.

c++/src/capnp/compiler/node-translator.c++

@@ -810,10 +810,7 @@ public:
       }
 
       schema2::Field::Builder fieldBuilder = member.getSchema();
-
-      fieldBuilder.setName(member.decl.getName().getValue());
       fieldBuilder.getOrdinal().setExplicit(entry.first);
-      fieldBuilder.setCodeOrder(member.codeOrder);
 
       switch (member.decl.getBody().which()) {
         case Declaration::Body::FIELD_DECL: {
@@ -891,7 +888,7 @@ public:
 
     // OK, we should have built all the members.  Now go through and make sure the discriminant
     // offsets have been copied over to the schemas and annotations have been applied.
-    root.setDiscriminantOffsetInSchema();
+    root.finishGroup();
     for (auto member: allMembers) {
       kj::StringPtr targetsFlagName;
       switch (member->decl.getBody().which()) {
@@ -900,13 +897,12 @@ public:
           break;
 
         case Declaration::Body::UNION_DECL:
-          member->setDiscriminantOffsetInSchema();
+          member->finishGroup();
           targetsFlagName = "targetsUnion";
           break;
 
         case Declaration::Body::GROUP_DECL:
-          member->setDiscriminantOffsetInSchema();  // in case it contains an unnamed union
-          member->node.setId(generateGroupId(member->parent->node.getId(), member->index));
+          member->finishGroup();
           targetsFlagName = "targetsGroup";
           break;
 
@@ -1026,6 +1022,8 @@ private:
         if (isInUnion) {
           builder.setDiscriminantValue(parent->unionDiscriminantCount++);
         }
+        builder.setName(decl.getName().getValue());
+        builder.setCodeOrder(codeOrder);
         schema = builder;
         return builder;
       }
@@ -1039,20 +1037,28 @@ private:
 
       auto structNode = node.getStruct();
       if (!structNode.hasFields()) {
+        if (parent != nullptr) {
           getSchema();  // Make sure field exists in parent once the first child is added.
+        }
         return structNode.initFields(childCount)[childInitializedCount++];
       } else {
         return structNode.getFields()[childInitializedCount++];
       }
     }
 
-    void setDiscriminantOffsetInSchema() {
+    void finishGroup() {
       if (unionScope != nullptr) {
         unionScope->addDiscriminant();  // if it hasn't happened already
         auto structNode = node.getStruct();
         structNode.setDiscriminantCount(unionDiscriminantCount);
         structNode.setDiscriminantOffset(KJ_ASSERT_NONNULL(unionScope->discriminantOffset));
       }
+
+      if (parent != nullptr) {
+        uint64_t groupId = generateGroupId(parent->node.getId(), index);
+        node.setId(groupId);
+        getSchema().setGroup(groupId);
+      }
     }
   };
 
@@ -1064,9 +1070,7 @@ private:
   // All members, including ones that don't have ordinals.
 
   void traverseUnion(List<Declaration>::Reader members, MemberInfo& parent,
-                     StructLayout::Union& layout) {
-    uint codeOrder = 0;
-
+                     StructLayout::Union& layout, uint& codeOrder) {
     if (members.size() < 2) {
       errorReporter.addErrorOn(parent.decl, "Union must have at least two members.");
     }
@@ -1145,8 +1149,11 @@ private:
         case Declaration::Body::UNION_DECL: {
           StructLayout::Union& unionLayout = arena.allocate<StructLayout::Union>(layout);
 
+          uint independentSubCodeOrder = 0;
+          uint* subCodeOrder = &independentSubCodeOrder;
           if (member.getName().getValue() == "") {
             memberInfo = &parent;
+            subCodeOrder = &codeOrder;
           } else {
             parent.childCount++;
             memberInfo = &arena.allocate<MemberInfo>(
@@ -1156,7 +1163,7 @@ private:
             allMembers.add(memberInfo);
           }
           memberInfo->unionScope = &unionLayout;
-          traverseUnion(member.getNestedDecls(), *memberInfo, unionLayout);
+          traverseUnion(member.getNestedDecls(), *memberInfo, unionLayout, *subCodeOrder);
           if (member.getId().which() == Declaration::Id::ORDINAL) {
             ordinal = member.getId().getOrdinal().getValue();
           }
@@ -1517,8 +1524,17 @@ void NodeTranslator::compileValue(ValueExpression::Reader source, schema2::Type:
                                   schema2::Value::Builder target, bool isBootstrap) {
 #warning "temporary hack for schema transition"
   switch (type.which()) {
+    case schema2::Type::TEXT:
+      target.setText(source.getBody().getString());
+      break;
+
+    case schema2::Type::UINT16:
+      target.setUint16(source.getBody().getPositiveInt());
+      break;
+
     default:
-      KJ_FAIL_ASSERT("Need to compile value type:", (uint)type.which());
+      KJ_FAIL_ASSERT("Need to compile value type:", (uint)type.which(),
+                     wipNode.getReader().getDisplayName());
   }
 
 #if 0

c++/src/capnp/generated-header-support.h

@@ -240,7 +240,8 @@ inline kj::StringTree structString(StructReader reader) {
 
 template <typename T>
 inline kj::StringTree unionString(StructReader reader) {
-  return unionString(reader, rawSchema<UnionParentType<T>>(), unionMemberIndex<T>());
+#warning "remove this"
+  return kj::strTree();
 }
 
 }  // namespace _ (private)

c++/src/capnp/schema-loader.c++

@@ -71,7 +71,8 @@ public:
   TryGetResult tryGet(uint64_t typeId) const;
   kj::Array<Schema> getAllLoaded() const;
 
-  void requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount);
+  void requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount,
+                         schema2::ElementSize preferredListEncoding);
   // Require any struct nodes loaded with this ID -- in the past and in the future -- to have at
   // least the given sizes.  Struct nodes that don't comply will simply be rewritten to comply.
   // This is used to ensure that parents of group nodes have at least the size of the group node,
@@ -86,6 +87,7 @@ private:
   struct RequiredSize {
     uint16_t dataWordCount;
     uint16_t pointerCount;
+    schema2::ElementSize preferredListEncoding;
   };
   std::unordered_map<uint64_t, RequiredSize> structSizeRequirements;
 
@@ -103,13 +105,15 @@ private:
   // (but at least can't cause memory corruption).
 
   kj::ArrayPtr<word> rewriteStructNodeWithSizes(
-      schema2::Node::Reader node, uint dataWordCount, uint pointerCount);
+      schema2::Node::Reader node, uint dataWordCount, uint pointerCount,
+      schema2::ElementSize preferredListEncoding);
   // Make a copy of the given node (which must be a struct node) and set its sizes to be the max
   // of what it said already and the given sizes.
 
   // If the encoded node does not meet the given struct size requirements, make a new copy that
   // does.
-  void applyStructSizeRequirement(_::RawSchema* raw, uint dataWordCount, uint pointerCount);
+  void applyStructSizeRequirement(_::RawSchema* raw, uint dataWordCount, uint pointerCount,
+                                  schema2::ElementSize preferredListEncoding);
 };
 
 // =======================================================================================
@@ -340,7 +344,8 @@ private:
       // Require that the group's scope has at least the same size as the group, so that anyone
       // constructing an instance of the outer scope can safely read/write the group.
       loader.requireStructSize(scopeId, structNode.getDataSectionWordSize(),
-                               structNode.getPointerSectionSize());
+                               structNode.getPointerSectionSize(),
+                               structNode.getPreferredListEncoding());
 
       // Require that the parent type is a struct.
       validateTypeId(scopeId, schema2::Node::STRUCT);
@@ -1131,7 +1136,8 @@ _::RawSchema* SchemaLoader::Impl::loadNative(const _::RawSchema* nativeSchema) {
     auto reqIter = structSizeRequirements.find(nativeSchema->id);
     if (reqIter != structSizeRequirements.end()) {
       applyStructSizeRequirement(result, reqIter->second.dataWordCount,
-                                 reqIter->second.pointerCount);
+                                 reqIter->second.pointerCount,
+                                 reqIter->second.preferredListEncoding);
     }
   } else {
     // The existing schema is newer.
@@ -1200,14 +1206,21 @@ kj::Array<Schema> SchemaLoader::Impl::getAllLoaded() const {
   return result;
 }
 
-void SchemaLoader::Impl::requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount) {
+void SchemaLoader::Impl::requireStructSize(uint64_t id, uint dataWordCount, uint pointerCount,
+                                           schema2::ElementSize preferredListEncoding) {
   auto& slot = structSizeRequirements[id];
   slot.dataWordCount = kj::max(slot.dataWordCount, dataWordCount);
   slot.pointerCount = kj::max(slot.pointerCount, pointerCount);
 
+  if (slot.dataWordCount + slot.pointerCount >= 2) {
+    slot.preferredListEncoding = schema2::ElementSize::INLINE_COMPOSITE;
+  } else {
+    slot.preferredListEncoding = kj::max(slot.preferredListEncoding, preferredListEncoding);
+  }
+
   auto iter = schemas.find(id);
   if (iter != schemas.end()) {
-    applyStructSizeRequirement(iter->second, dataWordCount, pointerCount);
+    applyStructSizeRequirement(iter->second, dataWordCount, pointerCount, preferredListEncoding);
   }
 }
 
@@ -1227,9 +1240,11 @@ kj::ArrayPtr<word> SchemaLoader::Impl::makeUncheckedNodeEnforcingSizeRequirement
       auto requirement = iter->second;
       auto structNode = node.getStruct();
       if (structNode.getDataSectionWordSize() < requirement.dataWordCount ||
-          structNode.getPointerSectionSize() < requirement.pointerCount) {
+          structNode.getPointerSectionSize() < requirement.pointerCount ||
+          structNode.getPreferredListEncoding() < requirement.preferredListEncoding) {
         return rewriteStructNodeWithSizes(node, requirement.dataWordCount,
-                                          requirement.pointerCount);
+                                          requirement.pointerCount,
+                                          requirement.preferredListEncoding);
       }
     }
   }
@@ -1238,7 +1253,8 @@ kj::ArrayPtr<word> SchemaLoader::Impl::makeUncheckedNodeEnforcingSizeRequirement
 }
 
 kj::ArrayPtr<word> SchemaLoader::Impl::rewriteStructNodeWithSizes(
-    schema2::Node::Reader node, uint dataWordCount, uint pointerCount) {
+    schema2::Node::Reader node, uint dataWordCount, uint pointerCount,
+    schema2::ElementSize preferredListEncoding) {
   MallocMessageBuilder builder;
   builder.setRoot(node);
 
@@ -1247,18 +1263,28 @@ kj::ArrayPtr<word> SchemaLoader::Impl::rewriteStructNodeWithSizes(
   newStruct.setDataSectionWordSize(kj::max(newStruct.getDataSectionWordSize(), dataWordCount));
   newStruct.setPointerSectionSize(kj::max(newStruct.getPointerSectionSize(), pointerCount));
 
+  if (newStruct.getDataSectionWordSize() + newStruct.getPointerSectionSize() >= 2) {
+    newStruct.setPreferredListEncoding(schema2::ElementSize::INLINE_COMPOSITE);
+  } else {
+    newStruct.setPreferredListEncoding(
+        kj::max(newStruct.getPreferredListEncoding(), preferredListEncoding));
+  }
+
   return makeUncheckedNode(root);
 }
 
 void SchemaLoader::Impl::applyStructSizeRequirement(
-    _::RawSchema* raw, uint dataWordCount, uint pointerCount) {
+    _::RawSchema* raw, uint dataWordCount, uint pointerCount,
+    schema2::ElementSize preferredListEncoding) {
   auto node = readMessageUnchecked<schema2::Node>(raw->encodedNode);
 
   auto structNode = node.getStruct();
   if (structNode.getDataSectionWordSize() < dataWordCount ||
-      structNode.getPointerSectionSize() < pointerCount) {
+      structNode.getPointerSectionSize() < pointerCount ||
+      structNode.getPreferredListEncoding() < preferredListEncoding) {
     // Sizes need to be increased.  Must rewrite.
-    kj::ArrayPtr<word> words = rewriteStructNodeWithSizes(node, dataWordCount, pointerCount);
+    kj::ArrayPtr<word> words = rewriteStructNodeWithSizes(
+        node, dataWordCount, pointerCount, preferredListEncoding);
 
     // We don't need to re-validate the node because we know this change could not possibly have
     // invalidated it.  Just remake the unchecked message.

c++/src/capnp/schema.h

@@ -154,8 +154,6 @@ private:
   friend class Schema;
   friend kj::StringTree _::structString(
       _::StructReader reader, const _::RawSchema& schema);
-  friend kj::StringTree _::unionString(
-      _::StructReader reader, const _::RawSchema& schema, uint fieldIndex);
 };
 
 class StructSchema::Field {

c++/src/capnp/schema2.capnp

@@ -42,7 +42,7 @@ struct Node {
   # If you want a shorter version of `displayName` (just naming this node, without its surrounding
   # scope), chop off this many characters from the beginning of `displayName`.
 
-  scopeId @3 :Id = 0;
+  scopeId @3 :Id;
   # ID of the lexical parent node.  Typically, the scope node will have a NestedNode pointing back
   # at this node, but robust code should avoid relying on this (and, in fact, group nodes are not
   # listed in the outer struct's nestedNodes, since they are listed in the fields).  `scopeId` is
@@ -105,7 +105,8 @@ struct Node {
       # `isGroup` = true.
 
       discriminantOffset @12 :UInt32;
-      # If `isUnion` is true, this is the offset of the union discriminant, in multiples of 16 bits.
+      # If `discriminantCount` is non-zero, this is the offset of the union discriminant, in
+      # multiples of 16 bits.
 
       fields @13 :List(Field);
       # Fields defined within this scope (either the struct's top-level fields, or the fields of

c++/src/capnp/stringify.c++

@@ -99,21 +99,19 @@ private:
   }
 };
 
-schema::Type::Body::Which whichMemberType(const StructSchema::Member& member) {
-  auto body = member.getProto().getBody();
-  switch (body.which()) {
-    case schema::StructNode::Member::Body::UNION_MEMBER:
-      return schema::Type::Body::VOID_TYPE;
-    case schema::StructNode::Member::Body::GROUP_MEMBER:
-      return schema::Type::Body::STRUCT_TYPE;
-    case schema::StructNode::Member::Body::FIELD_MEMBER:
-      return body.getFieldMember().getType().getBody().which();
+static schema2::Type::Which whichFieldType(const StructSchema::Field& field) {
+  auto proto = field.getProto();
+  switch (proto.which()) {
+    case schema2::Field::REGULAR:
+      return proto.getRegular().getType().which();
+    case schema2::Field::GROUP:
+      return schema2::Type::STRUCT;
   }
   KJ_UNREACHABLE;
 }
 
 static kj::StringTree print(const DynamicValue::Reader& value,
-                            schema::Type::Body::Which which, Indent indent,
+                            schema2::Type::Which which, Indent indent,
                             PrintMode mode) {
   switch (value.getType()) {
     case DynamicValue::UNKNOWN:
@@ -127,7 +125,7 @@ static kj::StringTree print(const DynamicValue::Reader& value,
     case DynamicValue::UINT:
       return kj::strTree(value.as<uint64_t>());
     case DynamicValue::FLOAT:
-      if (which == schema::Type::Body::FLOAT32_TYPE) {
+      if (which == schema2::Type::FLOAT32) {
         return kj::strTree(value.as<float>());
       } else {
         return kj::strTree(value.as<double>());
@@ -192,40 +190,31 @@ static kj::StringTree print(const DynamicValue::Reader& value,
     }
     case DynamicValue::STRUCT: {
       auto structValue = value.as<DynamicStruct>();
-      auto memberSchemas = structValue.getSchema().getMembers();
-
-      kj::Vector<kj::StringTree> printedMembers(memberSchemas.size());
-      for (auto member: memberSchemas) {
-        if (structValue.has(member)) {
-          auto name = member.getProto().getName();
-          if (name.size() == 0) {
-            // Unnamed union.  Just print the content.
-            printedMembers.add(kj::strTree(
-                print(structValue.get(member), whichMemberType(member), indent.next(), BARE)));
-          } else {
-            printedMembers.add(kj::strTree(
-                name, " = ",
-                print(structValue.get(member), whichMemberType(member), indent.next(), PREFIXED)));
-          }
+      auto unionFields = structValue.getSchema().getUnionFields();
+      auto nonUnionFields = structValue.getSchema().getNonUnionFields();
+
+      kj::Vector<kj::StringTree> printedFields(nonUnionFields.size() + (unionFields.size() != 0));
+
+      KJ_IF_MAYBE(field, structValue.which()) {
+        if (structValue.has(*field)) {
+          printedFields.add(kj::strTree(
+              field->getProto().getName(), " = ",
+              print(structValue.get(*field), whichFieldType(*field), indent.next(), PREFIXED)));
         }
       }
 
-      if (mode == PARENTHESIZED) {
-        return indent.delimit(printedMembers.releaseAsArray(), mode);
-      } else {
-        return kj::strTree('(', indent.delimit(printedMembers.releaseAsArray(), mode), ')');
+      for (auto field: nonUnionFields) {
+        if (structValue.has(field)) {
+          printedFields.add(kj::strTree(
+              field.getProto().getName(), " = ",
+              print(structValue.get(field), whichFieldType(field), indent.next(), PREFIXED)));
         }
       }
-    case DynamicValue::UNION: {
-      auto unionValue = value.as<DynamicUnion>();
-      KJ_IF_MAYBE(tag, unionValue.which()) {
-        return kj::strTree(
-            tag->getProto().getName(), '(',
-            print(unionValue.get(), whichMemberType(*tag), indent, PARENTHESIZED), ')');
+
+      if (mode == PARENTHESIZED) {
+        return indent.delimit(printedFields.releaseAsArray(), mode);
       } else {
-        // Unknown union member; must have come from newer
-        // version of the protocol.
-        return kj::strTree("<unknown union member>");
+        return kj::strTree('(', indent.delimit(printedFields.releaseAsArray(), mode), ')');
       }
     }
     case DynamicValue::INTERFACE:
@@ -240,17 +229,17 @@ static kj::StringTree print(const DynamicValue::Reader& value,
 }
 
 kj::StringTree stringify(DynamicValue::Reader value) {
-  return print(value, schema::Type::Body::STRUCT_TYPE, Indent(false), BARE);
+  return print(value, schema2::Type::STRUCT, Indent(false), BARE);
 }
 
 }  // namespace
 
 kj::StringTree prettyPrint(DynamicStruct::Reader value) {
-  return print(value, schema::Type::Body::STRUCT_TYPE, Indent(true), BARE);
+  return print(value, schema2::Type::STRUCT, Indent(true), BARE);
 }
 
 kj::StringTree prettyPrint(DynamicList::Reader value) {
-  return print(value, schema::Type::Body::LIST_TYPE, Indent(true), BARE);
+  return print(value, schema2::Type::LIST, Indent(true), BARE);
 }
 
 kj::StringTree prettyPrint(DynamicStruct::Builder value) { return prettyPrint(value.asReader()); }
@@ -260,8 +249,6 @@ kj::StringTree KJ_STRINGIFY(const DynamicValue::Reader& value) { return stringif
 kj::StringTree KJ_STRINGIFY(const DynamicValue::Builder& value) { return stringify(value.asReader()); }
 kj::StringTree KJ_STRINGIFY(DynamicEnum value) { return stringify(value); }
 kj::StringTree KJ_STRINGIFY(const DynamicObject& value) { return stringify(value); }
-kj::StringTree KJ_STRINGIFY(const DynamicUnion::Reader& value) { return stringify(value); }
-kj::StringTree KJ_STRINGIFY(const DynamicUnion::Builder& value) { return stringify(value.asReader()); }
 kj::StringTree KJ_STRINGIFY(const DynamicStruct::Reader& value) { return stringify(value); }
 kj::StringTree KJ_STRINGIFY(const DynamicStruct::Builder& value) { return stringify(value.asReader()); }
 kj::StringTree KJ_STRINGIFY(const DynamicList::Reader& value) { return stringify(value); }
@@ -273,11 +260,6 @@ kj::StringTree structString(StructReader reader, const RawSchema& schema) {
   return stringify(DynamicStruct::Reader(StructSchema(&schema), reader));
 }
 
-kj::StringTree unionString(StructReader reader, const RawSchema& schema, uint memberIndex) {
-  return stringify(DynamicUnion::Reader(
-      StructSchema(&schema).getMembers()[memberIndex].asUnion(), reader));
-}
-
 }  // namespace _ (private)
 
 }  // namespace capnp