Fix problems with AnyStruct / AnyList change.

c++/src/capnp/any.h

@@ -177,10 +177,11 @@ struct AnyPointer {
     inline BuilderFor<T> initAs(ListSchema schema, uint elementCount);
     // Only valid for T = DynamicList.  Requires `#include <capnp/dynamic.h>`.
 
-    inline AnyList::Builder initAsAnyList(_::FieldSize elementSize, uint elementCount);
+    inline AnyList::Builder initAsAnyList(ElementSize elementSize, uint elementCount);
     // Note: Does not accept INLINE_COMPOSITE for elementSize.
 
-    inline List<AnyStruct>::Builder initAsListOfAnyStruct(uint dataWordCount, uint pointerCount, uint elementCount);
+    inline List<AnyStruct>::Builder initAsListOfAnyStruct(
+        uint dataWordCount, uint pointerCount, uint elementCount);
 
     inline AnyStruct::Builder initAsAnyStruct(uint dataWordCount, uint pointerCount);
 
@@ -540,10 +541,8 @@ public:
   Reader() = default;
   inline Reader(_::ListReader reader): _reader(reader) {}
 
-  _::FieldSize getElementSize();
-  ElementCount size() {
-    return _reader.size();
-  }
+  inline ElementSize getElementSize() { return _reader.getElementSize(); }
+  inline uint size() { return _reader.size() / ELEMENTS; }
 
   template <typename T> ReaderFor<T> as() {
   // T must be List<U>.
@@ -556,13 +555,10 @@ private:
 class AnyList::Builder {
 public:
   inline Builder(decltype(nullptr)) {}
-  inline Builder(_::PointerBuilder builder, _::FieldSize size, const word* defaultValue = nullptr): _builder(builder.getList(size, defaultValue)) {}
   inline Builder(_::ListBuilder builder): _builder(builder) {}
 
-  _::FieldSize getElementSize();
-  ElementCount size() {
-    return _builder.size();
-  }
+  inline ElementSize getElementSize() { return _builder.getElementSize(); }
+  inline uint size() { return _builder.size() / ELEMENTS; }
 
   template <typename T> BuilderFor<T> as() {
   // T must be List<U>.
@@ -577,38 +573,48 @@ private:
 };
 
 namespace _ {  // (private)
+
 template <>
 struct PointerHelpers<AnyStruct, Kind::OTHER> {
-  static inline typename AnyStruct::Reader get(PointerReader reader, const word* defaultValue = nullptr) {
+  static inline typename AnyStruct::Reader get(
+      PointerReader reader, const word* defaultValue = nullptr) {
     return typename AnyStruct::Reader(reader.getStruct(defaultValue));
   }
-  static inline typename AnyStruct::Builder get(PointerBuilder builder,
-                                        const word* defaultValue = nullptr) {
-    return typename AnyStruct::Builder(builder, /* TODO: allow specifying the size! */ _::StructSize(0, 0), defaultValue);
+  static inline typename AnyStruct::Builder get(
+      PointerBuilder builder, const word* defaultValue = nullptr) {
+    // TODO(someday): allow specifying the size
+    return typename AnyStruct::Builder(
+        builder, _::StructSize(0 * WORDS, 0 * POINTERS), defaultValue);
   }
-  static inline typename AnyStruct::Builder init(PointerBuilder builder, uint dataWordCount, uint pointerCount) {
-    return typename AnyStruct::Builder(builder.initStruct(StructSize(dataWordCount, pointerCount)));
+  static inline typename AnyStruct::Builder init(
+      PointerBuilder builder, uint dataWordCount, uint pointerCount) {
+    return typename AnyStruct::Builder(builder.initStruct(
+        StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
   }
 };
+
 template <>
 struct PointerHelpers<AnyList, Kind::OTHER> {
-  static inline typename AnyList::Reader get(PointerReader reader, const word* defaultValue = nullptr) {
-    return typename AnyList::Reader(reader.getList(/* TODO: allow specifying the size! */ FieldSize::VOID, defaultValue));
+  static inline typename AnyList::Reader get(
+      PointerReader reader, const word* defaultValue = nullptr) {
+    return typename AnyList::Reader(reader.getListAnySize(defaultValue));
   }
-  static inline typename AnyList::Builder get(PointerBuilder builder,
-                                        const word* defaultValue = nullptr) {
-    return typename AnyList::Builder(builder, /* TODO: allow specifying the size! */ FieldSize::VOID, defaultValue);
+  static inline typename AnyList::Builder get(
+      PointerBuilder builder, const word* defaultValue = nullptr) {
+    return typename AnyList::Builder(builder.getListAnySize(defaultValue));
   }
-  static inline typename AnyList::Builder init(PointerBuilder builder, FieldSize elementSize, uint elementCount) {
-    return typename AnyList::Builder(builder.initList(elementSize, elementCount));
+  static inline typename AnyList::Builder init(
+      PointerBuilder builder, ElementSize elementSize, uint elementCount) {
+    return typename AnyList::Builder(builder.initList(elementSize, elementCount * ELEMENTS));
   }
-  static inline typename AnyList::Builder init(PointerBuilder builder, uint dataWordCount, uint pointerCount, uint elementCount) {
+  static inline typename AnyList::Builder init(
+      PointerBuilder builder, uint dataWordCount, uint pointerCount, uint elementCount) {
     return typename AnyList::Builder(builder.initStructList(
-        elementCount,
-        StructSize(dataWordCount, pointerCount)));
+        elementCount * ELEMENTS, StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
   }
 };
-} // end namespace _ (private)
+
+}  // namespace _ (private)
 
 // =======================================================================================
 // Pipeline helpers
@@ -701,14 +707,20 @@ inline BuilderFor<T> AnyPointer::Builder::initAs(uint elementCount) {
   return _::PointerHelpers<T>::init(builder, elementCount);
 }
 
-inline AnyList::Builder AnyPointer::Builder::initAsAnyList(_::FieldSize elementSize, uint elementCount) {
-  return _::PointerHelpers<AnyList>::init(builder, elementSize, elementCount);
+inline AnyList::Builder AnyPointer::Builder::initAsAnyList(
+    ElementSize elementSize, uint elementCount) {
+  return AnyList::Builder(builder.initList(elementSize, elementCount * ELEMENTS));
 }
 
-// inline List<AnyStruct>::Builder AnyPointer::Builder::initAsListOfAnyStruct(uint dataWordCount, uint pointerCount, uint elementCount);
+inline List<AnyStruct>::Builder AnyPointer::Builder::initAsListOfAnyStruct(
+    uint dataWordCount, uint pointerCount, uint elementCount) {
+  return List<AnyStruct>::Builder(builder.initStructList(elementCount * ELEMENTS,
+      _::StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
+}
 
 inline AnyStruct::Builder AnyPointer::Builder::initAsAnyStruct(uint dataWordCount, uint pointerCount) {
-  return _::PointerHelpers<AnyStruct>::init(builder, dataWordCount, pointerCount);
+  return AnyStruct::Builder(builder.initStruct(
+      _::StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
 }
 
 template <typename T>

c++/src/capnp/common.h

@@ -80,6 +80,21 @@ enum class Kind: uint8_t {
   // special handling. This includes types like AnyPointer, Dynamic*, etc.
 };
 
+enum class ElementSize: uint8_t {
+  // Size of a list element.
+
+  VOID = 0,
+  BIT = 1,
+  BYTE = 2,
+  TWO_BYTES = 3,
+  FOUR_BYTES = 4,
+  EIGHT_BYTES = 5,
+
+  POINTER = 6,
+
+  INLINE_COMPOSITE = 7
+};
+
 namespace schemas {
 
 template <typename T>

c++/src/capnp/layout.c++

@@ -1172,6 +1172,58 @@ struct WireHelpers {
     }
   }
 
+  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointerAnySize(
+      WirePointer* origRef, SegmentBuilder* origSegment, const word* defaultValue)) {
+    return getWritableListPointerAnySize(origRef, origRef->target(), origSegment, defaultValue);
+  }
+
+  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointerAnySize(
+      WirePointer* origRef, word* origRefTarget, SegmentBuilder* origSegment,
+      const word* defaultValue, BuilderArena* orphanArena = nullptr)) {
+    if (origRef->isNull()) {
+    useDefault:
+      if (defaultValue == nullptr ||
+          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
+        return ListBuilder();
+      }
+      origRefTarget = copyMessage(
+          origSegment, origRef, reinterpret_cast<const WirePointer*>(defaultValue));
+      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
+    }
+
+    WirePointer* ref = origRef;
+    SegmentBuilder* segment = origSegment;
+    word* ptr = followFars(ref, origRefTarget, segment);
+
+    KJ_REQUIRE(ref->kind() == WirePointer::LIST,
+        "Called getList{Field,Element}() but existing pointer is not a list.") {
+      goto useDefault;
+    }
+
+    FieldSize elementSize = ref->listRef.elementSize();
+
+    if (elementSize == FieldSize::INLINE_COMPOSITE) {
+      // Read the tag to get the actual element count.
+      WirePointer* tag = reinterpret_cast<WirePointer*>(ptr);
+      KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
+          "INLINE_COMPOSITE list with non-STRUCT elements not supported.");
+      ptr += POINTER_SIZE_IN_WORDS;
+
+      return ListBuilder(segment, ptr,
+                         tag->structRef.wordSize() * BITS_PER_WORD / ELEMENTS,
+                         tag->inlineCompositeListElementCount(),
+                         tag->structRef.dataSize.get() * BITS_PER_WORD,
+                         tag->structRef.ptrCount.get(), FieldSize::INLINE_COMPOSITE);
+    } else {
+      BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
+      WirePointerCount pointerCount = pointersPerElement(elementSize) * ELEMENTS;
+
+      auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
+      return ListBuilder(segment, ptr, step, ref->listRef.elementCount(),
+                         dataSize, pointerCount, elementSize);
+    }
+  }
+
   static KJ_ALWAYS_INLINE(ListBuilder getWritableStructListPointer(
       WirePointer* origRef, SegmentBuilder* origSegment, StructSize elementSize,
       const word* defaultValue)) {
@@ -1810,14 +1862,15 @@ struct WireHelpers {
 
   static KJ_ALWAYS_INLINE(ListReader readListPointer(
       SegmentReader* segment, const WirePointer* ref, const word* defaultValue,
-      FieldSize expectedElementSize, int nestingLimit)) {
+      FieldSize expectedElementSize, int nestingLimit, bool checkElementSize = true)) {
     return readListPointer(segment, ref, ref->target(), defaultValue,
-                           expectedElementSize, nestingLimit);
+                           expectedElementSize, nestingLimit, checkElementSize);
   }
 
   static KJ_ALWAYS_INLINE(ListReader readListPointer(
       SegmentReader* segment, const WirePointer* ref, const word* refTarget,
-      const word* defaultValue, FieldSize expectedElementSize, int nestingLimit)) {
+      const word* defaultValue, FieldSize expectedElementSize, int nestingLimit,
+      bool checkElementSize = true)) {
     if (ref->isNull()) {
     useDefault:
       if (defaultValue == nullptr ||
@@ -1875,6 +1928,7 @@ struct WireHelpers {
         goto useDefault;
       }
 
+      if (checkElementSize) {
         // If a struct list was not expected, then presumably a non-struct list was upgraded to a
         // struct list.  We need to manipulate the pointer to point at the first field of the
         // struct.  Together with the "stepBits", this will allow the struct list to be accessed as
@@ -1917,6 +1971,7 @@ struct WireHelpers {
           case FieldSize::INLINE_COMPOSITE:
             break;
         }
+      }
 
       return ListReader(
           segment, ptr, size, wordsPerElement * BITS_PER_WORD,
@@ -1938,6 +1993,7 @@ struct WireHelpers {
         goto useDefault;
       }
 
+      if (checkElementSize) {
         if (elementSize == FieldSize::BIT && expectedElementSize != FieldSize::BIT) {
           KJ_FAIL_REQUIRE(
               "Found bit list where struct list was expected; upgrading boolean lists to structs "
@@ -1964,6 +2020,7 @@ struct WireHelpers {
                    "Message contained list with incompatible element type.") {
           goto useDefault;
         }
+      }
 
       return ListReader(segment, ptr, ref->listRef.elementCount(), step,
                         dataSize, pointerCount, elementSize, nestingLimit - 1);
@@ -2094,6 +2151,10 @@ ListBuilder PointerBuilder::getStructList(StructSize elementSize, const word* de
   return WireHelpers::getWritableStructListPointer(pointer, segment, elementSize, defaultValue);
 }
 
+ListBuilder PointerBuilder::getListAnySize(const word* defaultValue) {
+  return WireHelpers::getWritableListPointerAnySize(pointer, segment, defaultValue);
+}
+
 template <>
 Text::Builder PointerBuilder::initBlob<Text>(ByteCount size) {
   return WireHelpers::initTextPointer(pointer, segment, size).value;
@@ -2157,16 +2218,14 @@ bool PointerBuilder::isNull() {
 }
 
 bool PointerBuilder::isStruct() {
-  word* refTarget;
   WirePointer* ptr = pointer;
-  WireHelpers::followFars(ptr, refTarget, segment);
+  WireHelpers::followFars(ptr, ptr->target(), segment);
   return ptr->kind() == WirePointer::Kind::STRUCT;
 }
 
 bool PointerBuilder::isList() {
-  word* refTarget;
   WirePointer* ptr = pointer;
-  WireHelpers::followFars(ptr, refTarget, segment);
+  WireHelpers::followFars(ptr, ptr->target(), segment);
   return ptr->kind() == WirePointer::Kind::LIST;
 }
 
@@ -2220,6 +2279,12 @@ ListReader PointerReader::getList(FieldSize expectedElementSize, const word* def
       segment, ref, defaultValue, expectedElementSize, nestingLimit);
 }
 
+ListReader PointerReader::getListAnySize(const word* defaultValue) const {
+  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
+  return WireHelpers::readListPointer(
+      segment, ref, defaultValue, FieldSize::VOID /* dummy */, nestingLimit, false);
+}
+
 template <>
 Text::Reader PointerReader::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) const {
   const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;

c++/src/capnp/layout.h

@@ -78,51 +78,10 @@ class BuilderArena;
 
 // =============================================================================
 
-enum class FieldSize: uint8_t {
-  // TODO(cleanup):  Rename to FieldLayout or maybe ValueLayout.
-
-  // Notice that each member of this enum, when representing a list element size, represents a
-  // size that is greater than or equal to the previous members, since INLINE_COMPOSITE is used
-  // only for multi-word structs.  This is important because it allows us to compare FieldSize
-  // values for the purpose of deciding when we need to upgrade a list.
-
-  VOID = 0,
-  BIT = 1,
-  BYTE = 2,
-  TWO_BYTES = 3,
-  FOUR_BYTES = 4,
-  EIGHT_BYTES = 5,
-
-  POINTER = 6,  // Indicates that the field lives in the pointer section, not the data section.
-
-  INLINE_COMPOSITE = 7
-  // A composite type of fixed width.  This serves two purposes:
-  // 1) For lists of composite types where all the elements would have the exact same width,
-  //    allocating a list of pointers which in turn point at the elements would waste space.  We
-  //    can avoid a layer of indirection by placing all the elements in a flat sequence, and only
-  //    indicating the element properties (e.g. field count for structs) once.
-  //
-  //    Specifically, a list pointer indicating INLINE_COMPOSITE element size actually points to
-  //    a "tag" describing one element.  This tag is formatted like a wire pointer, but the
-  //    "offset" instead stores the element count of the list.  The flat list of elements appears
-  //    immediately after the tag.  In the list pointer itself, the element count is replaced with
-  //    a word count for the whole list (excluding tag).  This allows the tag and elements to be
-  //    precached in a single step rather than two sequential steps.
-  //
-  //    It is NOT intended to be possible to substitute an INLINE_COMPOSITE list for a POINTER
-  //    list or vice-versa without breaking recipients.  Recipients expect one or the other
-  //    depending on the message definition.
-  //
-  //    However, it IS allowed to substitute an INLINE_COMPOSITE list -- specifically, of structs --
-  //    when a list was expected, or vice versa, with the assumption that the first field of the
-  //    struct (field number zero) correspond to the element type.  This allows a list of
-  //    primitives to be upgraded to a list of structs, avoiding the need to use parallel arrays
-  //    when you realize that you need to attach some extra information to each element of some
-  //    primitive list.
-  //
-  // 2) At one point there was a notion of "inline" struct fields, but it was deemed too much of
-  //    an implementation burden for too little gain, and so was deleted.
-};
+using FieldSize = capnp::ElementSize;
+// Legacy typedef.
+//
+// TODO(cleanup): Replace all uses.
 
 typedef decltype(BITS / ELEMENTS) BitsPerElement;
 typedef decltype(POINTERS / ELEMENTS) PointersPerElement;
@@ -330,6 +289,7 @@ public:
   StructBuilder getStruct(StructSize size, const word* defaultValue);
   ListBuilder getList(FieldSize elementSize, const word* defaultValue);
   ListBuilder getStructList(StructSize elementSize, const word* defaultValue);
+  ListBuilder getListAnySize(const word* defaultValue);
   template <typename T> typename T::Builder getBlob(const void* defaultValue,ByteCount defaultSize);
 #if !CAPNP_LITE
   kj::Own<ClientHook> getCapability();
@@ -409,6 +369,7 @@ public:
 
   StructReader getStruct(const word* defaultValue) const;
   ListReader getList(FieldSize expectedElementSize, const word* defaultValue) const;
+  ListReader getListAnySize(const word* defaultValue) const;
   template <typename T>
   typename T::Reader getBlob(const void* defaultValue, ByteCount defaultSize) const;
 #if !CAPNP_LITE
@@ -612,6 +573,8 @@ public:
     }
   }
 
+  inline ElementSize getElementSize() const { return elementSize; }
+
   inline ElementCount size() const;
   // The number of elements in the list.
 
@@ -680,6 +643,8 @@ public:
   inline ElementCount size() const;
   // The number of elements in the list.
 
+  inline ElementSize getElementSize() const { return elementSize; }
+
   Text::Reader asText();
   Data::Reader asData();
   // Reinterpret the list as a blob.  Throws an exception if the elements are not byte-sized.
@@ -854,7 +819,9 @@ inline Data::Builder StructBuilder::getDataSectionAsBlob() {
 }
 
 inline _::ListBuilder StructBuilder::getPointerSectionAsList() {
-  return _::ListBuilder(segment, pointers, pointerCount * BITS_PER_WORD / ELEMENTS, pointerCount, 0, 1, FieldSize::POINTER);
+  return _::ListBuilder(segment, pointers, pointerCount * BITS_PER_POINTER / ELEMENTS,
+                        pointerCount * (1 * ELEMENTS / POINTERS),
+                        0 * BITS, 1 * POINTERS, FieldSize::POINTER);
 }
 
 template <typename T>
@@ -936,7 +903,9 @@ inline Data::Reader StructReader::getDataSectionAsBlob() {
 }
 
 inline _::ListReader StructReader::getPointerSectionAsList() {
-  return _::ListReader(segment, pointers, pointerCount, pointerCount * BITS_PER_WORD / ELEMENTS, 0, 1, FieldSize::POINTER, nestingLimit);
+  return _::ListReader(segment, pointers, pointerCount * (1 * ELEMENTS / POINTERS),
+                       pointerCount * BITS_PER_POINTER / ELEMENTS,
+                       0 * BITS, 1 * POINTERS, FieldSize::POINTER, nestingLimit);
 }
 
 template <typename T>

c++/src/capnp/pointer-helpers.h

@@ -24,7 +24,6 @@
 
 #include "layout.h"
 #include "list.h"
-// #include "any.h"
 
 namespace capnp {
 namespace _ {  // private