Correctly handle inline fields that aren't present in the parent struct (i.e.…

Correctly handle inline fields that aren't present in the parent struct (i.e. because we're reading an older version of the struct that lacked those fields).

c++/src/capnproto/layout.c++

@@ -27,10 +27,19 @@
 #include "arena.h"
 #include <string.h>
 #include <limits>
+#include <stdlib.h>
 
 namespace capnproto {
 namespace internal {
 
+namespace {
+
+// This zero array is used only as a default value for inlined fields, which are limited
+// to no more than 64 words.
+static const AlignedData<64> ZERO = {{0}};
+
+}  // namespace
+
 // =======================================================================================
 
 struct WireReference {
@@ -193,14 +202,16 @@ static_assert(REFERENCES * BITS_PER_REFERENCE / BITS_PER_BYTE / BYTES == sizeof(
 struct WireHelpers {
   static CAPNPROTO_ALWAYS_INLINE(WordCount roundUpToWords(BitCount64 bits)) {
     static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
-    uint64_t bits2 = bits / BITS;
-    return ((bits2 >> 6) + ((bits2 & 63) != 0)) * WORDS;
+    return (bits + 63 * BITS) / BITS_PER_WORD;
   }
 
   static CAPNPROTO_ALWAYS_INLINE(WordCount roundUpToWords(ByteCount bytes)) {
     static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
-    uint bytes2 = bytes / BYTES;
-    return ((bytes2 >> 3) + ((bytes2 & 7) != 0)) * WORDS;
+    return (bytes + 7 * BYTES) / BYTES_PER_WORD;
+  }
+
+  static CAPNPROTO_ALWAYS_INLINE(ByteCount roundUpToBytes(BitCount bits)) {
+    return (bits + 7 * BITS) / BITS_PER_BYTE;
   }
 
   static CAPNPROTO_ALWAYS_INLINE(word* allocate(
@@ -944,6 +955,7 @@ struct WireHelpers {
 };
 
 // =======================================================================================
+// StructBuilder
 
 StructBuilder StructBuilder::initRoot(
     SegmentBuilder* segment, word* location, StructSize size) {
@@ -1022,6 +1034,9 @@ StructReader StructBuilder::asReader() const {
       0xffffffff * BYTES, 0xffff * REFERENCES, std::numeric_limits<int>::max());
 }
 
+// =======================================================================================
+// StructReader
+
 StructReader StructReader::readRootTrusted(const word* location) {
   return WireHelpers::readStructReference(nullptr, reinterpret_cast<const WireReference*>(location),
                                           nullptr, std::numeric_limits<int>::max());
@@ -1049,6 +1064,24 @@ StructReader StructReader::getStructField(
   return WireHelpers::readStructReference(segment, ref, defaultValue, nestingLimit);
 }
 
+StructReader StructReader::getInlineStructField(
+    ByteCount dataOffset, ByteCount inlineDataSize,
+    WireReferenceCount refIndex, WireReferenceCount inlineRefCount) const {
+  if (dataOffset + inlineDataSize <= dataSize &&
+      refIndex + inlineRefCount <= referenceCount) {
+    return StructReader(
+        segment, reinterpret_cast<const byte*>(data) + dataOffset,
+        // WireReference is incomplete here so we have to cast around...  Bah.
+        reinterpret_cast<const WireReference*>(
+            reinterpret_cast<const word*>(references) + refIndex * WORDS_PER_REFERENCE),
+        inlineDataSize, inlineRefCount,
+        nestingLimit);
+  } else {
+    // Return empty struct.
+    return StructReader();
+  }
+}
+
 ListReader StructReader::getListField(
     WireReferenceCount refIndex, FieldSize expectedElementSize, const word* defaultValue) const {
   const WireReference* ref = refIndex >= referenceCount ? nullptr : references + refIndex;
@@ -1056,6 +1089,52 @@ ListReader StructReader::getListField(
       segment, ref, defaultValue, expectedElementSize, nestingLimit);
 }
 
+ListReader StructReader::getInlineDataListField(
+    ByteCount offset, ElementCount elementCount, FieldSize elementSize) const {
+  if (offset + WireHelpers::roundUpToBytes(
+      elementCount * bitsPerElement(elementSize)) <= dataSize) {
+    return ListReader(
+        segment, reinterpret_cast<const byte*>(data) + offset, nullptr,
+        elementCount, bytesPerElement(elementSize), 0 * REFERENCES / ELEMENTS,
+        nestingLimit);
+  } else {
+    // Return zeros.
+    return ListReader(elementCount);
+  }
+}
+
+ListReader StructReader::getInlinePointerListField(
+    WireReferenceCount offset, ElementCount elementCount) const {
+  if (offset + elementCount * (1 * REFERENCES / ELEMENTS) <= referenceCount) {
+    return ListReader(
+        segment, nullptr,
+        reinterpret_cast<const WireReference*>(
+            reinterpret_cast<const word*>(references) + offset * WORDS_PER_REFERENCE),
+        elementCount, 0 * BYTES / ELEMENTS, 1 * REFERENCES / ELEMENTS,
+        nestingLimit);
+  } else {
+    // Return nulls.
+    return ListReader(elementCount);
+  }
+}
+
+ListReader StructReader::getInlineStructListField(
+    ByteCount dataOffset, WireReferenceCount ptrOffset, ElementCount elementCount,
+    StructSize elementSize) const {
+  if (dataOffset + elementSize.dataBytes <= dataSize &&
+      ptrOffset + elementSize.pointers <= referenceCount) {
+    return ListReader(
+        segment, reinterpret_cast<const byte*>(data) + dataOffset,
+        reinterpret_cast<const WireReference*>(
+            reinterpret_cast<const word*>(references) + ptrOffset * WORDS_PER_REFERENCE),
+        elementCount, elementSize.dataBytes / ELEMENTS, elementSize.pointers / ELEMENTS,
+        elementSize.dataBytes, elementSize.pointers, nestingLimit);
+  } else {
+    // Return empty structs.
+    return ListReader(elementCount);
+  }
+}
+
 Text::Reader StructReader::getTextField(
     WireReferenceCount refIndex, const void* defaultValue, ByteCount defaultSize) const {
   const WireReference* ref = refIndex >= referenceCount ? nullptr : references + refIndex;
@@ -1068,6 +1147,20 @@ Data::Reader StructReader::getDataField(
   return WireHelpers::readDataReference(segment, ref, defaultValue, defaultSize);
 }
 
+Data::Reader StructReader::getInlineDataField(ByteCount offset, ByteCount size) const {
+  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
+  if (offset + size <= dataSize) {
+    return Data::Reader(reinterpret_cast<const char*>(reinterpret_cast<const byte*>(data) + offset),
+                        size / BYTES);
+  } else {
+    CHECK(size < sizeof(ZERO) * BYTES);
+    return Data::Reader(reinterpret_cast<const char*>(ZERO.bytes), size / BYTES);
+  }
+}
+
+// =======================================================================================
+// ListBuilder
+
 StructBuilder ListBuilder::getStructElement(ElementCount index, StructSize elementSize) const {
   // TODO:  Inline this method?
   ByteCount indexByte = ElementCount64(index) * stepBytes;
@@ -1134,6 +1227,15 @@ ListReader ListBuilder::asReader(StructSize elementSize) const {
                     elementSize.dataBytes, elementSize.pointers, std::numeric_limits<int>::max());
 }
 
+// =======================================================================================
+// ListReader
+
+ListReader::ListReader(ElementCount elementCount)
+    : segment(nullptr), data(ZERO.bytes),
+      pointers(reinterpret_cast<const WireReference*>(data)), elementCount(elementCount),
+      stepBytes(0 * BYTES / ELEMENTS), stepPointers(0 * REFERENCES / ELEMENTS),
+      structDataSize(0), structReferenceCount(0), nestingLimit(0) {}
+
 StructReader ListReader::getStructElement(ElementCount index) const {
   // TODO:  Inline this method?
   VALIDATE_INPUT((segment == nullptr) | (nestingLimit > 0),

c++/src/capnproto/layout.h

@@ -422,9 +422,9 @@ public:
   // struct reference, which in turn points at the struct value.  The default value is allowed to
   // be null, in which case an empty struct is used.
 
-  CAPNPROTO_ALWAYS_INLINE(StructReader getInlineStructField(
+  StructReader getInlineStructField(
       ByteCount dataOffset, ByteCount inlineDataSize,
-      WireReferenceCount refIndex, WireReferenceCount inlineRefCount) const);
+      WireReferenceCount refIndex, WireReferenceCount inlineRefCount) const;
   // Gets an inlined struct field, given the position and size of the data and pointer sections.
 
   ListReader getListField(WireReferenceCount refIndex, FieldSize expectedElementSize,
@@ -432,17 +432,17 @@ public:
   // Get the list field at the given index in the reference segment, or the default value if not
   // initialized.  The default value is allowed to be null, in which case an empty list is used.
 
-  CAPNPROTO_ALWAYS_INLINE(ListReader getInlineDataListField(
-      ByteCount offset, ElementCount elementCount, FieldSize elementSize) const);
+  ListReader getInlineDataListField(
+      ByteCount offset, ElementCount elementCount, FieldSize elementSize) const;
   // Get an inline list field.
 
-  CAPNPROTO_ALWAYS_INLINE(ListReader getInlinePointerListField(
-      WireReferenceCount offset, ElementCount elementCount) const);
+  ListReader getInlinePointerListField(
+      WireReferenceCount offset, ElementCount elementCount) const;
   // Get an inline list field.
 
-  CAPNPROTO_ALWAYS_INLINE(ListReader getInlineStructListField(
+  ListReader getInlineStructListField(
       ByteCount dataOffset, WireReferenceCount ptrOffset, ElementCount elementCount,
-      StructSize elementSize) const);
+      StructSize elementSize) const;
   // Get an inline struct list field.
 
   Text::Reader getTextField(WireReferenceCount refIndex,
@@ -453,8 +453,7 @@ public:
                             const void* defaultValue, ByteCount defaultSize) const;
   // Gets the data field, or the given default value if not initialized.
 
-  CAPNPROTO_ALWAYS_INLINE(Data::Reader getInlineDataField(
-      ByteCount offset, ByteCount size) const);
+  Data::Reader getInlineDataField(ByteCount offset, ByteCount size) const;
   // Gets the inline data field.
 
   WireReferenceCount getReferenceCount() { return referenceCount; }
@@ -577,6 +576,10 @@ public:
         stepBytes(0 * BYTES / ELEMENTS), stepPointers(0 * REFERENCES / ELEMENTS),
         structDataSize(0), structReferenceCount(0), nestingLimit(0) {}
 
+  ListReader(ElementCount elementCount);
+  // Constructs a ListReader representing a list where all elements are zero.  Intended to be used
+  // only for inlined lists that are out-of-bounds in the parent struct.
+
   inline ElementCount size();
   // The number of elements in the list.
 
@@ -810,58 +813,6 @@ T StructReader::getDataField(ElementCount offset, typename MaskType<T>::Type mas
   return unmask<T>(getDataField<typename MaskType<T>::Type>(offset), mask);
 }
 
-inline StructReader StructReader::getInlineStructField(
-    ByteCount dataOffset, ByteCount inlineDataSize,
-    WireReferenceCount refIndex, WireReferenceCount inlineRefCount) const {
-  // TODO(soon):  Too complicated to be inlined?
-  return StructReader(
-      segment, reinterpret_cast<const byte*>(data) + dataOffset,
-      // WireReference is incomplete here so we have to cast around...  Bah.
-      reinterpret_cast<const WireReference*>(
-          reinterpret_cast<const word*>(references) + refIndex * WORDS_PER_REFERENCE),
-      inlineDataSize * (dataOffset + inlineDataSize <= dataSize),
-      inlineRefCount * (refIndex + inlineRefCount <= referenceCount),
-      nestingLimit);
-}
-
-inline ListReader StructReader::getInlineDataListField(
-    ByteCount offset, ElementCount elementCount, FieldSize elementSize) const {
-  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
-  return ListReader(
-      segment, reinterpret_cast<const byte*>(data) + offset, nullptr,
-      elementCount, bytesPerElement(elementSize), 0 * REFERENCES / ELEMENTS,
-      nestingLimit);
-}
-
-inline ListReader StructReader::getInlinePointerListField(
-    WireReferenceCount offset, ElementCount elementCount) const {
-  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
-  return ListReader(
-      segment, nullptr,
-      reinterpret_cast<const WireReference*>(
-          reinterpret_cast<const word*>(references) + offset * WORDS_PER_REFERENCE),
-      elementCount, 0 * BYTES / ELEMENTS, 1 * REFERENCES / ELEMENTS,
-      nestingLimit);
-}
-
-inline ListReader StructReader::getInlineStructListField(
-    ByteCount dataOffset, WireReferenceCount ptrOffset, ElementCount elementCount,
-    StructSize elementSize) const {
-  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
-  return ListReader(
-      segment, reinterpret_cast<const byte*>(data) + dataOffset,
-      reinterpret_cast<const WireReference*>(
-          reinterpret_cast<const word*>(references) + ptrOffset * WORDS_PER_REFERENCE),
-      elementCount, elementSize.dataBytes / ELEMENTS, elementSize.pointers / ELEMENTS,
-      elementSize.dataBytes, elementSize.pointers, nestingLimit);
-}
-
-inline Data::Reader StructReader::getInlineDataField(ByteCount offset, ByteCount size) const {
-  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
-  return Data::Reader(reinterpret_cast<const char*>(reinterpret_cast<const byte*>(data) + offset),
-                      size / BYTES);
-}
-
 // -------------------------------------------------------------------
 
 inline ElementCount ListBuilder::size() { return elementCount; }

c++/src/capnproto/test-util.c++

@@ -1152,6 +1152,139 @@ void genericCheckInlineDefaults(Reader reader) {
   }
 }
 
+template <typename Builder>
+void genericInitEmptyInlineLists(Builder builder) {
+  // Initialize all inline lists in TestInlineDefaults to "empty" values, of the same sizes that
+  // genericInitInlineDefaults() would create.
+
+  builder.initNormal();
+
+  {
+    auto lists = builder.initLists();
+
+    ASSERT_EQ(2u, lists.getVoidList().size());
+    ASSERT_EQ(3u, lists.getBoolList().size());
+    ASSERT_EQ(4u, lists.getUInt8List().size());
+    ASSERT_EQ(5u, lists.getUInt16List().size());
+    ASSERT_EQ(6u, lists.getUInt32List().size());
+    ASSERT_EQ(7u, lists.getUInt64List().size());
+    ASSERT_EQ(8u, lists.getTextList().size());
+
+    ASSERT_EQ(2u, lists.getStructList0().size());
+    ASSERT_EQ(3u, lists.getStructList1().size());
+    ASSERT_EQ(4u, lists.getStructList8().size());
+    ASSERT_EQ(2u, lists.getStructList16().size());
+    ASSERT_EQ(3u, lists.getStructList32().size());
+    ASSERT_EQ(4u, lists.getStructList64().size());
+    ASSERT_EQ(2u, lists.getStructList128().size());
+    ASSERT_EQ(3u, lists.getStructList192().size());
+
+    ASSERT_EQ(4u, lists.getStructList0p().size());
+    ASSERT_EQ(2u, lists.getStructList1p().size());
+    ASSERT_EQ(3u, lists.getStructList8p().size());
+    ASSERT_EQ(4u, lists.getStructList16p().size());
+    ASSERT_EQ(2u, lists.getStructList32p().size());
+    ASSERT_EQ(3u, lists.getStructList64p().size());
+    ASSERT_EQ(4u, lists.getStructList128p().size());
+    ASSERT_EQ(2u, lists.getStructList192p().size());
+  }
+
+  {
+    auto sl = builder.initStructLists();
+
+    sl.initList0(2);
+    sl.initList1(2);
+    sl.initList8(2);
+    sl.initList16(2);
+    sl.initList32(2);
+    sl.initList64(2);
+    sl.initListP(2);
+  }
+
+  {
+    auto ll = builder.initListLists();
+
+    {
+      auto l = ll.initInt32ListList(3);
+      l.init(0, 3);
+      l.init(1, 2);
+      l.init(2, 1);
+    }
+
+    {
+      auto l = ll.initTextListList(3);
+      l.init(0, 2);
+      l.init(1, 1);
+      l.init(2, 2);
+    }
+
+    {
+      auto l = ll.initStructListList(2);
+      l.init(0, 2);
+      l.init(1, 1);
+    }
+
+    {
+      auto l = ll.initInt32InlineListList(2);
+      EXPECT_EQ(7u, l[0].size());
+      EXPECT_EQ(7u, l[1].size());
+    }
+
+    {
+      auto l = ll.initTextInlineListList(3);
+      EXPECT_EQ(5u, l[0].size());
+      EXPECT_EQ(5u, l[1].size());
+      EXPECT_EQ(5u, l[2].size());
+    }
+
+    {
+      auto l = ll.initStructInlineListList(3);
+      EXPECT_EQ(3u, l[0].size());
+      EXPECT_EQ(3u, l[1].size());
+      EXPECT_EQ(3u, l[2].size());
+    }
+
+    ll.initInlineDataList(5);
+
+    {
+      auto l = ll.initInt32InlineListListList(3);
+      l.init(0, 3);
+      l.init(1, 2);
+      l.init(2, 1);
+      EXPECT_EQ(2u, l[0][0].size());
+      EXPECT_EQ(2u, l[0][1].size());
+      EXPECT_EQ(2u, l[0][2].size());
+      EXPECT_EQ(2u, l[1][0].size());
+      EXPECT_EQ(2u, l[1][1].size());
+      EXPECT_EQ(2u, l[2][0].size());
+    }
+
+    {
+      auto l = ll.initTextInlineListListList(2);
+      l.init(0, 2);
+      l.init(1, 1);
+      EXPECT_EQ(5u, l[0][0].size());
+      EXPECT_EQ(5u, l[0][1].size());
+      EXPECT_EQ(5u, l[1][0].size());
+    }
+
+    {
+      auto l = ll.initStructInlineListListList(2);
+      l.init(0, 2);
+      l.init(1, 1);
+      EXPECT_EQ(3u, l[0][0].size());
+      EXPECT_EQ(3u, l[0][1].size());
+      EXPECT_EQ(3u, l[1][0].size());
+    }
+
+    {
+      auto l = ll.initInlineDataListList(2);
+      l.init(0, 3);
+      l.init(1, 2);
+    }
+  }
+}
+
 }  // namespace
 
 void initTestMessage(TestAllTypes::Builder builder) { genericInitTestMessage(builder); }

compiler/src/Compiler.hs

@@ -752,6 +752,13 @@ enforceFixed (Just (Located pos (requestedDataSize, requestedPointerCount)))
             \backwards-compatibility."
             actualPointerCount requestedPointerCount
 
+    recover () $ when (dataSectionBits actualDataSize > maxStructDataWords * 64) $
+        makeError pos $ printf "Struct is too big.  Maximum data section size is %d bytes."
+            (maxStructDataWords * 8)
+    recover () $ when (actualPointerCount > maxStructPointers) $
+        makeError pos $ printf "Struct is too big.  Maximum pointer section size is %d."
+            maxStructPointers
+
     return (validatedRequestedDataSize, requestedPointerCount)
 
 ------------------------------------------------------------------------------------------
@@ -891,6 +898,9 @@ compileDecl scope
                 DescUnion u -> Just (u, unionFieldDiscriminantMap u ! number)
                 _ -> Nothing
         typeDesc <- compileType scope typeExp
+        recover () $ when (fieldSizeInBits (fieldSize typeDesc) > maxInlineFieldBits) $
+            makeError pos $ printf "Inlined fields cannot exceed %d bytes."
+            (div maxInlineFieldBits 8)
         defaultDesc <- case defaultValue of
             Just (Located defaultPos value) -> do
                 result <- fmap Just (compileValue defaultPos typeDesc value)

compiler/src/Semantics.hs

@@ -39,6 +39,14 @@ import Grammar(AnnotationTarget(..))
 -- ordinal is 65534.
 maxOrdinal = 65534 :: Integer
 
+-- Inline fields can be 64 words.  (This limit is relied upon by implementations which may need
+-- to produce some sort of default value when an inlined field is not actually present in the
+-- struct.)
+maxInlineFieldBits = 64 * 64 :: Integer
+
+maxStructDataWords = 65536 :: Integer
+maxStructPointers = 65536 :: Integer
+
 type ByteString = [Word8]
 
 data Desc = DescFile FileDesc
@@ -290,6 +298,11 @@ data FieldSize = SizeVoid
                | SizeReference
                | SizeInlineComposite DataSectionSize Integer
 
+fieldSizeInBits SizeVoid = 0
+fieldSizeInBits (SizeData d) = dataSizeInBits d
+fieldSizeInBits SizeReference = 64
+fieldSizeInBits (SizeInlineComposite ds pc) = dataSectionBits ds + pc * 64
+
 data FieldOffset = VoidOffset
                  | DataOffset DataSize Integer
                  | PointerOffset Integer
@@ -361,7 +374,7 @@ typeName scope (InlineStructType desc) = descQualifiedName scope (DescStruct des
 typeName scope (InterfaceType desc) = descQualifiedName scope (DescInterface desc)
 typeName scope (ListType t) = "List(" ++ typeName scope t ++ ")"
 typeName scope (InlineListType t s) = printf "InlineList(%s, %d)" (typeName scope t) s
-typeName scope (InlineDataType s) = printf "InlineData(%d)" s
+typeName _ (InlineDataType s) = printf "InlineData(%d)" s
 
 -- Computes the qualified name for the given descriptor within the given scope.
 -- At present the scope is only used to determine whether the target is in the same file.  If