Restore ability to upgrade from List(Bool) to List(T) where T is a struct whose…

Restore ability to upgrade from List(Bool) to List(T) where T is a struct whose @0 field is of type Bool.  I previously disallowed this to reduce complexity, but it turned out to actually increase complexity.

c++/src/capnproto/encoding-test.c++

@@ -435,7 +435,8 @@ TEST(Encoding, SmallStructLists) {
   EXPECT_EQ(0u, sl.getStructListList().size());
 
   { auto l = sl.initList0 (2); l[0].setF(Void::VOID);        l[1].setF(Void::VOID); }
-  { auto l = sl.initList1 (2); l[0].setF(true);              l[1].setF(false); }
+  { auto l = sl.initList1 (4); l[0].setF(true);              l[1].setF(false);
+                               l[2].setF(true);              l[3].setF(true); }
   { auto l = sl.initList8 (2); l[0].setF(123u);              l[1].setF(45u); }
   { auto l = sl.initList16(2); l[0].setF(12345u);            l[1].setF(6789u); }
   { auto l = sl.initList32(2); l[0].setF(123456789u);        l[1].setF(234567890u); }
@@ -482,6 +483,139 @@ TEST(Encoding, SmallStructLists) {
   }
 }
 
+// =======================================================================================
+
+TEST(Encoding, ListUpgrade) {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<test::TestObject>();
+
+  root.initObjectField<List<uint16_t>>(3).copyFrom({12, 34, 56});
+
+  checkList(root.getObjectField<List<uint8_t>>(), {12, 34, 56});
+
+  {
+    auto l = root.getObjectField<List<test::TestLists::Struct8>>();
+    ASSERT_EQ(3u, l.size());
+    EXPECT_EQ(12u, l[0].getF());
+    EXPECT_EQ(34u, l[1].getF());
+    EXPECT_EQ(56u, l[2].getF());
+  }
+
+  checkList(root.getObjectField<List<uint16_t>>(), {12, 34, 56});
+
+  auto reader = root.asReader();
+
+  checkList(reader.getObjectField<List<uint8_t>>(), {12, 34, 56});
+
+  {
+    auto l = reader.getObjectField<List<test::TestLists::Struct8>>();
+    ASSERT_EQ(3u, l.size());
+    EXPECT_EQ(12u, l[0].getF());
+    EXPECT_EQ(34u, l[1].getF());
+    EXPECT_EQ(56u, l[2].getF());
+  }
+
+  try {
+    reader.getObjectField<List<uint32_t>>();
+    ADD_FAILURE() << "Expected exception.";
+  } catch (const Exception& e) {
+    // expected
+  }
+
+  {
+    auto l = reader.getObjectField<List<test::TestLists::Struct32>>();
+    ASSERT_EQ(3u, l.size());
+
+    // These should return default values because the structs aren't big enough.
+    EXPECT_EQ(0u, l[0].getF());
+    EXPECT_EQ(0u, l[1].getF());
+    EXPECT_EQ(0u, l[2].getF());
+  }
+
+  checkList(reader.getObjectField<List<uint16_t>>(), {12, 34, 56});
+}
+
+TEST(Encoding, BitListDowngrade) {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<test::TestObject>();
+
+  root.initObjectField<List<uint16_t>>(4).copyFrom({0x1201u, 0x3400u, 0x5601u, 0x7801u});
+
+  checkList(root.getObjectField<List<bool>>(), {true, false, true, true});
+
+  {
+    auto l = root.getObjectField<List<test::TestLists::Struct1>>();
+    ASSERT_EQ(4u, l.size());
+    EXPECT_TRUE(l[0].getF());
+    EXPECT_FALSE(l[1].getF());
+    EXPECT_TRUE(l[2].getF());
+    EXPECT_TRUE(l[3].getF());
+  }
+
+  checkList(root.getObjectField<List<uint16_t>>(), {0x1201u, 0x3400u, 0x5601u, 0x7801u});
+
+  auto reader = root.asReader();
+
+  checkList(reader.getObjectField<List<bool>>(), {true, false, true, true});
+
+  {
+    auto l = reader.getObjectField<List<test::TestLists::Struct1>>();
+    ASSERT_EQ(4u, l.size());
+    EXPECT_TRUE(l[0].getF());
+    EXPECT_FALSE(l[1].getF());
+    EXPECT_TRUE(l[2].getF());
+    EXPECT_TRUE(l[3].getF());
+  }
+
+  checkList(reader.getObjectField<List<uint16_t>>(), {0x1201u, 0x3400u, 0x5601u, 0x7801u});
+}
+
+TEST(Encoding, BitListUpgrade) {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<test::TestObject>();
+
+  root.initObjectField<List<bool>>(4).copyFrom({true, false, true, true});
+
+  {
+    auto l = root.getObjectField<List<test::TestFieldZeroIsBit>>();
+    ASSERT_EQ(4u, l.size());
+    EXPECT_TRUE(l[0].getBit());
+    EXPECT_FALSE(l[1].getBit());
+    EXPECT_TRUE(l[2].getBit());
+    EXPECT_TRUE(l[3].getBit());
+  }
+
+  auto reader = root.asReader();
+
+  try {
+    reader.getObjectField<List<uint8_t>>();
+    ADD_FAILURE() << "Expected exception.";
+  } catch (const Exception& e) {
+    // expected
+  }
+
+  {
+    auto l = reader.getObjectField<List<test::TestFieldZeroIsBit>>();
+    ASSERT_EQ(4u, l.size());
+    EXPECT_TRUE(l[0].getBit());
+    EXPECT_FALSE(l[1].getBit());
+    EXPECT_TRUE(l[2].getBit());
+    EXPECT_TRUE(l[3].getBit());
+
+    // Other fields are defaulted.
+    EXPECT_TRUE(l[0].getSecondBit());
+    EXPECT_TRUE(l[1].getSecondBit());
+    EXPECT_TRUE(l[2].getSecondBit());
+    EXPECT_TRUE(l[3].getSecondBit());
+    EXPECT_EQ(123u, l[0].getThirdField());
+    EXPECT_EQ(123u, l[1].getThirdField());
+    EXPECT_EQ(123u, l[2].getThirdField());
+    EXPECT_EQ(123u, l[3].getThirdField());
+  }
+
+  checkList(reader.getObjectField<List<bool>>(), {true, false, true, true});
+}
+
 // =======================================================================================
 // Tests of generated code, not really of the encoding.
 // TODO(cleanup):  Move to a different test?

c++/src/capnproto/layout.c++

@@ -343,7 +343,7 @@ struct WireHelpers {
           case FieldSize::EIGHT_BYTES: {
             WordCount wordCount = roundUpToWords(
                 ElementCount64(src->listRef.elementCount()) *
-                bitsPerElement(src->listRef.elementSize()));
+                dataBitsPerElement(src->listRef.elementSize()));
             const word* srcPtr = src->target();
             word* dstPtr = allocate(dst, segment, wordCount, WireReference::LIST);
             memcpy(dstPtr, srcPtr, wordCount * BYTES_PER_WORD / BYTES);
@@ -418,7 +418,8 @@ struct WireHelpers {
     ref->structRef.set(size);
 
     // Build the StructBuilder.
-    return StructBuilder(segment, ptr, reinterpret_cast<WireReference*>(ptr + size.data));
+    return StructBuilder(segment, ptr, reinterpret_cast<WireReference*>(ptr + size.data),
+                         size.data * BITS_PER_WORD, size.pointers, 0 * BITS);
   }
 
   static CAPNPROTO_ALWAYS_INLINE(StructBuilder getWritableStructReference(
@@ -445,7 +446,8 @@ struct WireHelpers {
           "Trying to update struct with incorrect reference count.");
     }
 
-    return StructBuilder(segment, ptr, reinterpret_cast<WireReference*>(ptr + size.data));
+    return StructBuilder(segment, ptr, reinterpret_cast<WireReference*>(ptr + size.data),
+                         size.data * BITS_PER_WORD, size.pointers, 0 * BITS);
   }
 
   static CAPNPROTO_ALWAYS_INLINE(ListBuilder initListReference(
@@ -454,9 +456,9 @@ struct WireHelpers {
     DPRECOND(elementSize != FieldSize::INLINE_COMPOSITE,
         "Should have called initStructListReference() instead.");
 
-    // Note:  Use bitsPerElement(), not bytesPerElement(), to handle bit lists correctly when
-    //   computing wordCount.  After that, the step is not used for bit lists.
-    auto step = bitsPerElement(elementSize);
+    BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
+    WireReferenceCount referenceCount = pointersPerElement(elementSize) * ELEMENTS;
+    auto step = (dataSize + referenceCount * BITS_PER_REFERENCE) / ELEMENTS;
 
     // Calculate size of the list.
     WordCount wordCount = roundUpToWords(ElementCount64(elementCount) * step);
@@ -468,18 +470,15 @@ struct WireHelpers {
     ref->listRef.set(elementSize, elementCount);
 
     // Build the ListBuilder.
-    return ListBuilder(segment, ptr, step / BITS_PER_BYTE, elementCount);
+    return ListBuilder(segment, ptr, step, elementCount, dataSize, referenceCount);
   }
 
   static CAPNPROTO_ALWAYS_INLINE(ListBuilder initStructListReference(
       WireReference* ref, SegmentBuilder* segment, ElementCount elementCount,
       StructSize elementSize)) {
     if (elementSize.preferredListEncoding != FieldSize::INLINE_COMPOSITE) {
-      // Small data-only struct.  Allocate a list of primitives instead.  Don't ever pack as
-      // individual bits, though; we don't support that.
-      return initListReference(ref, segment, elementCount,
-                               elementSize.preferredListEncoding == FieldSize::BIT ?
-                                   FieldSize::BYTE : elementSize.preferredListEncoding);
+      // Small data-only struct.  Allocate a list of primitives instead.
+      return initListReference(ref, segment, elementCount, elementSize.preferredListEncoding);
     }
 
     auto wordsPerElement = elementSize.total() / ELEMENTS;
@@ -499,7 +498,8 @@ struct WireHelpers {
     ptr += REFERENCE_SIZE_IN_WORDS;
 
     // Build the ListBuilder.
-    return ListBuilder(segment, ptr, wordsPerElement * BYTES_PER_WORD, elementCount);
+    return ListBuilder(segment, ptr, wordsPerElement * BITS_PER_WORD, elementCount,
+                       elementSize.data * BITS_PER_WORD, elementSize.pointers);
   }
 
   static CAPNPROTO_ALWAYS_INLINE(ListBuilder getWritableListReference(
@@ -526,11 +526,15 @@ struct WireHelpers {
           "INLINE_COMPOSITE list with non-STRUCT elements not supported.");
 
       // First list element is at tag + 1 reference.
-      return ListBuilder(segment, tag + 1, tag->structRef.wordSize() * BYTES_PER_WORD / ELEMENTS,
-                         tag->inlineCompositeListElementCount());
+      return ListBuilder(segment, tag + 1, tag->structRef.wordSize() * BITS_PER_WORD / ELEMENTS,
+                         tag->inlineCompositeListElementCount(),
+                         tag->structRef.dataSize.get() * BITS_PER_WORD,
+                         tag->structRef.refCount.get());
     } else {
-      auto step = bytesPerElement(ref->listRef.elementSize());
-      return ListBuilder(segment, ptr, step, ref->listRef.elementCount());
+      BitCount dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
+      WireReferenceCount referenceCount = pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
+      auto step = (dataSize + referenceCount * BITS_PER_REFERENCE) / ELEMENTS;
+      return ListBuilder(segment, ptr, step, ref->listRef.elementCount(), dataSize, referenceCount);
     }
   }
 
@@ -633,21 +637,25 @@ struct WireHelpers {
 
         // First list element is at tag + 1 reference.
         return ObjectBuilder(
-            ListBuilder(segment, tag + 1, tag->structRef.wordSize() * BYTES_PER_WORD / ELEMENTS,
-                        tag->inlineCompositeListElementCount()),
-            FieldSize::INLINE_COMPOSITE);
+            ListBuilder(segment, tag + 1, tag->structRef.wordSize() * BITS_PER_WORD / ELEMENTS,
+                        tag->inlineCompositeListElementCount(),
+                        tag->structRef.dataSize.get() * BITS_PER_WORD,
+                        tag->structRef.refCount.get()));
       } else {
-        auto step = bytesPerElement(ref->listRef.elementSize());
-        return ObjectBuilder(
-            ListBuilder(segment, ptr, step, ref->listRef.elementCount()),
-            ref->listRef.elementSize());
+        BitCount dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
+        WireReferenceCount referenceCount =
+            pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
+        auto step = (dataSize + referenceCount * BITS_PER_REFERENCE) / ELEMENTS;
+        return ObjectBuilder(ListBuilder(
+            segment, ptr, step, ref->listRef.elementCount(), dataSize, referenceCount));
       }
     } else {
-      return ObjectBuilder(
-          StructBuilder(segment, ptr,
-                        reinterpret_cast<WireReference*>(ptr + ref->structRef.dataSize.get())),
-          ref->structRef.dataSize.get() * BYTES_PER_WORD,
-          ref->structRef.refCount.get());
+      return ObjectBuilder(StructBuilder(
+          segment, ptr,
+          reinterpret_cast<WireReference*>(ptr + ref->structRef.dataSize.get()),
+          ref->structRef.dataSize.get() * BITS_PER_WORD,
+          ref->structRef.refCount.get(),
+          0 * BITS));
     }
   }
 
@@ -661,7 +669,7 @@ struct WireHelpers {
     if (ref == nullptr || ref->isNull()) {
     useDefault:
       if (defaultValue == nullptr) {
-        return StructReader(nullptr, nullptr, nullptr, 0 * BYTES, 0 * REFERENCES,
+        return StructReader(nullptr, nullptr, nullptr, 0 * BITS, 0 * REFERENCES, 0 * BITS,
                             std::numeric_limits<int>::max());
       }
       segment = nullptr;
@@ -695,9 +703,9 @@ struct WireHelpers {
 
     return StructReader(
         segment, ptr, reinterpret_cast<const WireReference*>(ptr + ref->structRef.dataSize.get()),
-        ref->structRef.dataSize.get() * BYTES_PER_WORD,
+        ref->structRef.dataSize.get() * BITS_PER_WORD,
         ref->structRef.refCount.get(),
-        nestingLimit - 1);
+        0 * BITS, nestingLimit - 1);
   }
 
   static CAPNPROTO_ALWAYS_INLINE(ListReader readListReference(
@@ -815,13 +823,17 @@ struct WireHelpers {
       }
 
       return ListReader(
-          segment, ptr, size, wordsPerElement * BYTES_PER_WORD,
-          tag->structRef.dataSize.get() * BYTES_PER_WORD,
+          segment, ptr, size, wordsPerElement * BITS_PER_WORD,
+          tag->structRef.dataSize.get() * BITS_PER_WORD,
           tag->structRef.refCount.get(), nestingLimit - 1);
 
     } else {
-      // The elements of the list are NOT structs.
-      auto step = bitsPerElement(ref->listRef.elementSize());
+      // This is a primitive or pointer list, but all such lists can also be interpreted as struct
+      // lists.  We need to compute the data size and reference count for such structs.
+      BitCount dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
+      WireReferenceCount referenceCount =
+          pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
+      auto step = (dataSize + referenceCount * BITS_PER_REFERENCE) / ELEMENTS;
 
       if (segment != nullptr) {
         VALIDATE_INPUT(segment->containsInterval(ptr, ptr +
@@ -831,43 +843,29 @@ struct WireHelpers {
         }
       }
 
-      if (ref->listRef.elementSize() == expectedElementSize) {
-        return ListReader(segment, ptr, ref->listRef.elementCount(), step / BITS_PER_BYTE,
-                          nestingLimit - 1);
-      } else if (expectedElementSize == FieldSize::INLINE_COMPOSITE) {
-        // We were expecting a struct list, but we received a list of some other type.  Perhaps a
-        // non-struct list was recently upgraded to a struct list, but the sender is using the
-        // old version of the protocol.  We need to verify that the struct's first field matches
-        // what the sender sent us.
-
-        ByteCount dataSize = 0 * BYTES;
-        WireReferenceCount referenceCount = 0 * REFERENCES;
-
-        switch (ref->listRef.elementSize()) {
-          case FieldSize::VOID: break;
-          case FieldSize::BYTE: dataSize = 1 * BYTES; break;
-          case FieldSize::TWO_BYTES: dataSize = 2 * BYTES; break;
-          case FieldSize::FOUR_BYTES: dataSize = 4 * BYTES; break;
-          case FieldSize::EIGHT_BYTES: dataSize = 8 * BYTES; break;
-          case FieldSize::REFERENCE: referenceCount = 1 * REFERENCES; break;
-
-          case FieldSize::BIT:
-            FAIL_VALIDATE_INPUT("Message contained a bit list where a struct list was expected.") {
-              dataSize = 0 * BYTES;
-            }
-            break;
-
-          case FieldSize::INLINE_COMPOSITE:
-            FAIL_CHECK();
-            break;
+      if (segment != nullptr) {
+        // Verify that the elements are at least as large as the expected type.  Note that if we
+        // expected INLINE_COMPOSITE, the expected sizes here will be zero, because bounds checking
+        // will be performed at field access time.  So this check here is for the case where we
+        // expected a list of some primitive or pointer type.
+
+        BitCount expectedDataBitsPerElement =
+            dataBitsPerElement(expectedElementSize) * ELEMENTS;
+        WireReferenceCount expectedPointersPerElement =
+            pointersPerElement(expectedElementSize) * ELEMENTS;
+
+        VALIDATE_INPUT(expectedDataBitsPerElement <= dataSize,
+            "Message contained list with incompatible element type.") {
+          goto useDefault;
+        }
+        VALIDATE_INPUT(expectedPointersPerElement <= referenceCount,
+            "Message contained list with incompatible element type.") {
+          goto useDefault;
         }
-
-        return ListReader(segment, ptr, ref->listRef.elementCount(), step / BITS_PER_BYTE,
-                          dataSize, referenceCount, nestingLimit - 1);
-      } else {
-        PRECOND(segment != nullptr, "Trusted message had incompatible list element type.");
-        goto useDefault;
       }
+
+      return ListReader(segment, ptr, ref->listRef.elementCount(), step,
+                        dataSize, referenceCount, nestingLimit - 1);
     }
   }
 
@@ -1003,9 +1001,9 @@ struct WireHelpers {
         return ObjectReader(
             StructReader(segment, ptr,
                          reinterpret_cast<const WireReference*>(ptr + ref->structRef.dataSize.get()),
-                         ref->structRef.dataSize.get() * BYTES_PER_WORD,
+                         ref->structRef.dataSize.get() * BITS_PER_WORD,
                          ref->structRef.refCount.get(),
-                         nestingLimit - 1));
+                         0 * BITS, nestingLimit - 1));
       case WireReference::LIST: {
         FieldSize elementSize = ref->listRef.elementSize();
 
@@ -1043,12 +1041,13 @@ struct WireHelpers {
           }
 
           return ObjectReader(
-              ListReader(segment, ptr, elementCount, wordsPerElement * BYTES_PER_WORD,
-                         tag->structRef.dataSize.get() * BYTES_PER_WORD,
-                         tag->structRef.refCount.get(), nestingLimit - 1),
-              elementSize);
+              ListReader(segment, ptr, elementCount, wordsPerElement * BITS_PER_WORD,
+                         tag->structRef.dataSize.get() * BITS_PER_WORD,
+                         tag->structRef.refCount.get(), nestingLimit - 1));
         } else {
-          decltype(BITS / ELEMENTS) step = bitsPerElement(elementSize);
+          BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
+          WireReferenceCount referenceCount = pointersPerElement(elementSize) * ELEMENTS;
+          auto step = (dataSize + referenceCount * BITS_PER_REFERENCE) / ELEMENTS;
           ElementCount elementCount = ref->listRef.elementCount();
           WordCount wordCount = roundUpToWords(ElementCount64(elementCount) * step);
 
@@ -1060,11 +1059,8 @@ struct WireHelpers {
           }
 
           return ObjectReader(
-              ListReader(segment, ptr, elementCount, step / BITS_PER_BYTE,
-                  elementSize == FieldSize::REFERENCE ? 0 * BYTES : step * ELEMENTS / BITS_PER_BYTE,
-                  elementSize == FieldSize::REFERENCE ? 1 * REFERENCES : 0 * REFERENCES,
-                  nestingLimit - 1),
-              elementSize);
+              ListReader(segment, ptr, elementCount, step, dataSize, referenceCount,
+                         nestingLimit - 1));
         }
       }
       default:
@@ -1149,14 +1145,8 @@ ObjectBuilder StructBuilder::getObjectField(
 }
 
 StructReader StructBuilder::asReader() const {
-  // HACK:  We just give maxed-out data size and reference counts because they are only
-  // used for checking for field presence.
-  static_assert(sizeof(WireReference::structRef.dataSize) == 2,
-      "Has the maximum data size changed?");
-  static_assert(sizeof(WireReference::structRef.refCount) == 2,
-      "Has the maximum reference count changed?");
   return StructReader(segment, data, references,
-      0xffffffff * BYTES, 0xffff * REFERENCES, std::numeric_limits<int>::max());
+      dataSize, referenceCount, bit0Offset, std::numeric_limits<int>::max());
 }
 
 // =======================================================================================
@@ -1178,11 +1168,6 @@ StructReader StructReader::readRoot(
                                           nullptr, nestingLimit);
 }
 
-StructReader StructReader::readEmpty() {
-  return StructReader(nullptr, nullptr, nullptr, 0 * BYTES, 0 * REFERENCES,
-                      std::numeric_limits<int>::max());
-}
-
 StructReader StructReader::getStructField(
     WireReferenceCount refIndex, const word* defaultValue) const {
   const WireReference* ref = refIndex >= referenceCount ? nullptr : references + refIndex;
@@ -1218,80 +1203,66 @@ ObjectReader StructReader::getObjectField(
 // ListBuilder
 
 StructBuilder ListBuilder::getStructElement(ElementCount index, StructSize elementSize) const {
-  // TODO:  Inline this method?
-  byte* structData = ptr + index * stepBytes;
+  BitCount64 indexBit = ElementCount64(index) * step;
+  byte* structData = ptr + indexBit / BITS_PER_BYTE;
   return StructBuilder(segment, structData,
-      reinterpret_cast<WireReference*>(structData) + elementSize.data / WORDS_PER_REFERENCE);
+      reinterpret_cast<WireReference*>(structData) + elementSize.data / WORDS_PER_REFERENCE,
+      structDataSize, structReferenceCount, indexBit % BITS_PER_BYTE);
 }
 
 ListBuilder ListBuilder::initListElement(
     ElementCount index, FieldSize elementSize, ElementCount elementCount) const {
   return WireHelpers::initListReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes),
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE),
       segment, elementCount, elementSize);
 }
 
 ListBuilder ListBuilder::initStructListElement(
     ElementCount index, ElementCount elementCount, StructSize elementSize) const {
   return WireHelpers::initStructListReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes),
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE),
       segment, elementCount, elementSize);
 }
 
 ListBuilder ListBuilder::getListElement(ElementCount index) const {
   return WireHelpers::getWritableListReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, nullptr);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, nullptr);
 }
 
 Text::Builder ListBuilder::initTextElement(ElementCount index, ByteCount size) const {
   return WireHelpers::initTextReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, size);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, size);
 }
 void ListBuilder::setTextElement(ElementCount index, Text::Reader value) const {
   WireHelpers::setTextReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, value);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, value);
 }
 Text::Builder ListBuilder::getTextElement(ElementCount index) const {
   return WireHelpers::getWritableTextReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, "", 0 * BYTES);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, "", 0 * BYTES);
 }
 
 Data::Builder ListBuilder::initDataElement(ElementCount index, ByteCount size) const {
   return WireHelpers::initDataReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, size);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, size);
 }
 void ListBuilder::setDataElement(ElementCount index, Data::Reader value) const {
   WireHelpers::setDataReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, value);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, value);
 }
 Data::Builder ListBuilder::getDataElement(ElementCount index) const {
   return WireHelpers::getWritableDataReference(
-      reinterpret_cast<WireReference*>(ptr + index * stepBytes), segment, nullptr, 0 * BYTES);
+      reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), segment, nullptr,
+      0 * BYTES);
 }
 
 ObjectBuilder ListBuilder::getObjectElement(ElementCount index, const word* defaultValue) const {
   return WireHelpers::getWritableObjectReference(
-      segment, reinterpret_cast<WireReference*>(ptr + index * stepBytes), defaultValue);
-}
-
-ListReader ListBuilder::asReader(FieldSize elementSize) const {
-  // TODO:  For INLINE_COMPOSITE I suppose we could just check the tag?
-  PRECOND(elementSize != FieldSize::INLINE_COMPOSITE,
-      "Need to call the other asReader() overload for INLINE_COMPOSITE lists.");
-  return ListReader(segment, ptr, elementCount, stepBytes, std::numeric_limits<int>::max());
+      segment, reinterpret_cast<WireReference*>(ptr + index * step / BITS_PER_BYTE), defaultValue);
 }
 
-ListReader ListBuilder::asReader(StructSize elementSize) const {
-  DCHECK(stepBytes * ELEMENTS >= elementSize.data * BYTES_PER_WORD ||
-         (elementSize.data == 1 * WORDS &&
-          elementSize.preferredListEncoding != FieldSize::INLINE_COMPOSITE &&
-          elementSize.preferredListEncoding != FieldSize::REFERENCE &&
-          stepBytes == bytesPerElement(elementSize.preferredListEncoding)),
-         "Assumptions used here did not hold.");
-
-  return ListReader(segment, ptr, elementCount, stepBytes,
-                    std::min(stepBytes * ELEMENTS, elementSize.data * BYTES_PER_WORD),
-                    elementSize.pointers,
+ListReader ListBuilder::asReader() const {
+  return ListReader(segment, ptr, elementCount, step, structDataSize, structReferenceCount,
                     std::numeric_limits<int>::max());
 }
 
@@ -1301,13 +1272,13 @@ ListReader ListBuilder::asReader(StructSize elementSize) const {
 StructReader ListReader::getStructElement(ElementCount index) const {
   VALIDATE_INPUT((segment == nullptr) | (nestingLimit > 0),
         "Message is too deeply-nested or contains cycles.  See capnproto::ReadOptions.") {
-    return StructReader::readEmpty();
+    return StructReader();
   }
 
-  ByteCount indexByte = index * stepBytes;
-  const byte* structData = ptr + indexByte;
+  BitCount64 indexBit = ElementCount64(index) * step;
+  const byte* structData = ptr + indexBit / BITS_PER_BYTE;
   const WireReference* structPointers =
-      reinterpret_cast<const WireReference*>(structData + structDataSize);
+      reinterpret_cast<const WireReference*>(structData + structDataSize / BITS_PER_BYTE);
 
   // This check should pass if there are no bugs in the list pointer validation code.
   DCHECK(structReferenceCount == 0 * REFERENCES ||
@@ -1316,7 +1287,8 @@ StructReader ListReader::getStructElement(ElementCount index) const {
 
   return StructReader(
       segment, structData, structPointers,
-      structDataSize, structReferenceCount, nestingLimit - 1);
+      structDataSize, structReferenceCount,
+      indexBit % BITS_PER_BYTE, nestingLimit - 1);
 }
 
 static const WireReference* checkAlignment(const void* ptr) {
@@ -1328,25 +1300,25 @@ static const WireReference* checkAlignment(const void* ptr) {
 ListReader ListReader::getListElement(
     ElementCount index, FieldSize expectedElementSize) const {
   return WireHelpers::readListReference(
-      segment, checkAlignment(ptr + index * stepBytes),
+      segment, checkAlignment(ptr + index * step / BITS_PER_BYTE),
       nullptr, expectedElementSize, nestingLimit);
 }
 
 Text::Reader ListReader::getTextElement(ElementCount index) const {
   return WireHelpers::readTextReference(
-      segment, checkAlignment(ptr + index * stepBytes),
+      segment, checkAlignment(ptr + index * step / BITS_PER_BYTE),
       "", 0 * BYTES);
 }
 
 Data::Reader ListReader::getDataElement(ElementCount index) const {
   return WireHelpers::readDataReference(
-      segment, checkAlignment(ptr + index * stepBytes),
+      segment, checkAlignment(ptr + index * step / BITS_PER_BYTE),
       nullptr, 0 * BYTES);
 }
 
 ObjectReader ListReader::getObjectElement(ElementCount index, const word* defaultValue) const {
   return WireHelpers::readObjectReference(
-      segment, checkAlignment(ptr + index * stepBytes), defaultValue, nestingLimit);
+      segment, checkAlignment(ptr + index * step / BITS_PER_BYTE), defaultValue, nestingLimit);
 }
 
 }  // namespace internal

c++/src/capnproto/layout.h

@@ -94,7 +94,7 @@ enum class FieldSize: uint8_t {
 };
 
 typedef decltype(BITS / ELEMENTS) BitsPerElement;
-typedef decltype(BYTES / ELEMENTS) BytesPerElement;
+typedef decltype(REFERENCES / ELEMENTS) PointersPerElement;
 
 namespace internal {
   static constexpr BitsPerElement BITS_PER_ELEMENT_TABLE[8] = {
@@ -104,20 +104,17 @@ namespace internal {
       16 * BITS / ELEMENTS,
       32 * BITS / ELEMENTS,
       64 * BITS / ELEMENTS,
-      64 * BITS / ELEMENTS,
+      0 * BITS / ELEMENTS,
       0 * BITS / ELEMENTS
   };
 }
 
-inline constexpr BitsPerElement bitsPerElement(FieldSize size) {
+inline constexpr BitsPerElement dataBitsPerElement(FieldSize size) {
   return internal::BITS_PER_ELEMENT_TABLE[static_cast<int>(size)];
 }
 
-inline constexpr BytesPerElement bytesPerElement(FieldSize size) {
-  // BIT gets rounded down to zero bytes.  This is OK because bytesPerElement() is only used in
-  // cases where this doesn't matter, e.g. for computing stepBytes which is ignored by bit ops
-  // anyway.
-  return bitsPerElement(size) / BITS_PER_BYTE;
+inline constexpr PointersPerElement pointersPerElement(FieldSize size) {
+  return size == FieldSize::REFERENCE ? 1 * REFERENCES / ELEMENTS : 0 * REFERENCES / ELEMENTS;
 }
 
 template <int wordCount>
@@ -249,7 +246,7 @@ private:
 
 class StructBuilder {
 public:
-  inline StructBuilder(): segment(nullptr), data(nullptr), references(nullptr) {}
+  inline StructBuilder(): segment(nullptr), data(nullptr), references(nullptr), bit0Offset(0) {}
 
   static StructBuilder initRoot(SegmentBuilder* segment, word* location, StructSize size);
   static StructBuilder getRoot(SegmentBuilder* segment, word* location, StructSize size);
@@ -331,8 +328,21 @@ private:
   void* data;                  // Pointer to the encoded data.
   WireReference* references;   // Pointer to the encoded references.
 
-  inline StructBuilder(SegmentBuilder* segment, void* data, WireReference* references)
-      : segment(segment), data(data), references(references) {}
+  BitCount32 dataSize;
+  // Size of data segment.  We use a bit count rather than a word count to more easily handle the
+  // case of struct lists encoded with less than a word per element.
+
+  WireReferenceCount16 referenceCount;  // Size of the reference segment.
+
+  BitCount8 bit0Offset;
+  // A special hack:  If dataSize == 1 bit, then bit0Offset is the offset of that bit within the
+  // byte pointed to by `data`.  In all other cases, this is zero.  This is needed to implement
+  // struct lists where each struct is one bit.
+
+  inline StructBuilder(SegmentBuilder* segment, void* data, WireReference* references,
+                       BitCount dataSize, WireReferenceCount referenceCount, BitCount8 bit0Offset)
+      : segment(segment), data(data), references(references),
+        dataSize(dataSize), referenceCount(referenceCount), bit0Offset(bit0Offset) {}
 
   friend class ListBuilder;
   friend struct WireHelpers;
@@ -342,11 +352,10 @@ class StructReader {
 public:
   inline StructReader()
       : segment(nullptr), data(nullptr), references(nullptr), dataSize(0),
-        referenceCount(0), nestingLimit(0) {}
+        referenceCount(0), bit0Offset(0), nestingLimit(0) {}
 
   static StructReader readRootTrusted(const word* location);
   static StructReader readRoot(const word* location, SegmentReader* segment, int nestingLimit);
-  static StructReader readEmpty();
 
   template <typename T>
   CAPNPROTO_ALWAYS_INLINE(T getDataField(ElementCount offset) const);
@@ -390,21 +399,32 @@ private:
   const void* data;
   const WireReference* references;
 
-  ByteCount32 dataSize;
-  // Size of data segment.  We use a byte count rather than a word count to more easily handle the
+  BitCount32 dataSize;
+  // Size of data segment.  We use a bit count rather than a word count to more easily handle the
   // case of struct lists encoded with less than a word per element.
 
   WireReferenceCount16 referenceCount;  // Size of the reference segment.
 
+  BitCount8 bit0Offset;
+  // A special hack:  If dataSize == 1 bit, then bit0Offset is the offset of that bit within the
+  // byte pointed to by `data`.  In all other cases, this is zero.  This is needed to implement
+  // struct lists where each struct is one bit.
+  //
+  // TODO(someday):  Consider packing this together with dataSize, since we have 10 extra bits
+  //   there doing nothing -- or arguably 12 bits, if you consider that 2-bit and 4-bit sizes
+  //   aren't allowed.  Consider that we could have a method like getDataSizeIn<T>() which is
+  //   specialized to perform the correct shifts for each size.
+
   int nestingLimit;
   // Limits the depth of message structures to guard against stack-overflow-based DoS attacks.
   // Once this reaches zero, further pointers will be pruned.
   // TODO:  Limit to 8 bits for better alignment?
 
   inline StructReader(SegmentReader* segment, const void* data, const WireReference* references,
-                      ByteCount dataSize, WireReferenceCount referenceCount, int nestingLimit)
+                      BitCount dataSize, WireReferenceCount referenceCount, BitCount8 bit0Offset,
+                      int nestingLimit)
       : segment(segment), data(data), references(references),
-        dataSize(dataSize), referenceCount(referenceCount),
+        dataSize(dataSize), referenceCount(referenceCount), bit0Offset(bit0Offset),
         nestingLimit(nestingLimit) {}
 
   friend class ListReader;
@@ -418,7 +438,7 @@ class ListBuilder {
 public:
   inline ListBuilder()
       : segment(nullptr), ptr(nullptr), elementCount(0 * ELEMENTS),
-        stepBytes(0 * BYTES / ELEMENTS) {}
+        step(0 * BITS / ELEMENTS) {}
 
   inline ElementCount size();
   // The number of elements in the list.
@@ -467,11 +487,8 @@ public:
   ObjectBuilder getObjectElement(ElementCount index, const word* defaultValue) const;
   // Gets a pointer element of arbitrary type.
 
-  ListReader asReader(FieldSize elementSize) const;
-  // Get a ListReader pointing at the same memory.  Use this version only for non-struct lists.
-
-  ListReader asReader(StructSize elementSize) const;
-  // Get a ListReader pointing at the same memory.  Use this version only for struct lists.
+  ListReader asReader() const;
+  // Get a ListReader pointing at the same memory.
 
 private:
   SegmentBuilder* segment;  // Memory segment in which the list resides.
@@ -480,14 +497,20 @@ private:
 
   ElementCount elementCount;  // Number of elements in the list.
 
-  decltype(BYTES / ELEMENTS) stepBytes;
+  decltype(BITS / ELEMENTS) step;
   // The distance between elements.
-  // Bit lists ignore stepBytes -- they are always tightly-packed.
+
+  BitCount32 structDataSize;
+  WireReferenceCount16 structReferenceCount;
+  // The struct properties to use when interpreting the elements as structs.  All lists can be
+  // interpreted as struct lists, so these are always filled in.
 
   inline ListBuilder(SegmentBuilder* segment, void* ptr,
-                     decltype(BYTES / ELEMENTS) stepBytes, ElementCount size)
+                     decltype(BITS / ELEMENTS) step, ElementCount size,
+                     BitCount structDataSize, WireReferenceCount structReferenceCount)
       : segment(segment), ptr(reinterpret_cast<byte*>(ptr)),
-        elementCount(size), stepBytes(stepBytes) {}
+        elementCount(size), step(step), structDataSize(structDataSize),
+        structReferenceCount(structReferenceCount) {}
 
   friend class StructBuilder;
   friend struct WireHelpers;
@@ -496,7 +519,7 @@ private:
 class ListReader {
 public:
   inline ListReader()
-      : segment(nullptr), ptr(nullptr), elementCount(0), stepBytes(0 * BYTES / ELEMENTS),
+      : segment(nullptr), ptr(nullptr), elementCount(0), step(0 * BITS / ELEMENTS),
         structDataSize(0), structReferenceCount(0), nestingLimit(0) {}
 
   inline ElementCount size();
@@ -528,30 +551,24 @@ private:
 
   ElementCount elementCount;  // Number of elements in the list.
 
-  decltype(BYTES / ELEMENTS) stepBytes;
+  decltype(BITS / ELEMENTS) step;
   // The distance between elements.
-  // Bit lists ignore stepBytes -- they are always tightly-packed.
 
-  ByteCount structDataSize;
-  WireReferenceCount structReferenceCount;
-  // If the elements are structs, the properties of the struct.
+  BitCount32 structDataSize;
+  WireReferenceCount16 structReferenceCount;
+  // The struct properties to use when interpreting the elements as structs.  All lists can be
+  // interpreted as struct lists, so these are always filled in.
 
   int nestingLimit;
   // Limits the depth of message structures to guard against stack-overflow-based DoS attacks.
   // Once this reaches zero, further pointers will be pruned.
 
   inline ListReader(SegmentReader* segment, const void* ptr,
-                    ElementCount elementCount, decltype(BYTES / ELEMENTS) stepBytes,
+                    ElementCount elementCount, decltype(BITS / ELEMENTS) step,
+                    BitCount structDataSize, WireReferenceCount structReferenceCount,
                     int nestingLimit)
       : segment(segment), ptr(reinterpret_cast<const byte*>(ptr)), elementCount(elementCount),
-        stepBytes(stepBytes), structDataSize(0), structReferenceCount(0),
-        nestingLimit(nestingLimit) {}
-  inline ListReader(SegmentReader* segment, const void* ptr,
-                    ElementCount elementCount, decltype(BYTES / ELEMENTS) stepBytes,
-                    ByteCount structDataSize, WireReferenceCount structReferenceCount,
-                    int nestingLimit)
-      : segment(segment), ptr(reinterpret_cast<const byte*>(ptr)), elementCount(elementCount),
-        stepBytes(stepBytes), structDataSize(structDataSize),
+        step(step), structDataSize(structDataSize),
         structReferenceCount(structReferenceCount), nestingLimit(nestingLimit) {}
 
   friend class StructReader;
@@ -572,27 +589,16 @@ struct ObjectBuilder {
 
   Kind kind;
 
-  FieldSize listElementSize;
-  // Only set if kind == LIST.  This would be part of the union, except that then ObjectReader would
-  // end up larger overall.
-
-  WireReferenceCount16 structPointerSectionSize;
-  ByteCount32 structDataSectionSize;
-  // For kind == STRUCT, the size of the struct.  For kind == LIST, the size of each element of the
-  // list, unless listElementSize == BIT.
-
   union {
     StructBuilder structBuilder;
     ListBuilder listBuilder;
   };
 
   ObjectBuilder(): kind(NULL_POINTER), structBuilder() {}
-  ObjectBuilder(StructBuilder structBuilder, ByteCount32 dataSectionSize,
-                WireReferenceCount16 pointerSectionSize)
-      : kind(STRUCT), structPointerSectionSize(pointerSectionSize),
-        structDataSectionSize(dataSectionSize), structBuilder(structBuilder) {}
-  ObjectBuilder(ListBuilder listBuilderBuilder, FieldSize elementSize)
-      : kind(LIST), listElementSize(elementSize), listBuilder(listBuilder) {}
+  ObjectBuilder(StructBuilder structBuilder)
+      : kind(STRUCT), structBuilder(structBuilder) {}
+  ObjectBuilder(ListBuilder listBuilderBuilder)
+      : kind(LIST), listBuilder(listBuilder) {}
 };
 
 struct ObjectReader {
@@ -606,10 +612,6 @@ struct ObjectReader {
 
   Kind kind;
 
-  FieldSize listElementSize;
-  // Only set if kind == LIST.  This would be part of the union, except that then ObjectReader would
-  // end up larger overall.
-
   union {
     StructReader structReader;
     ListReader listReader;
@@ -618,8 +620,8 @@ struct ObjectReader {
   ObjectReader(): kind(NULL_POINTER), structReader() {}
   ObjectReader(StructReader structReader)
       : kind(STRUCT), structReader(structReader) {}
-  ObjectReader(ListReader listReader, FieldSize elementSize)
-      : kind(LIST), listElementSize(elementSize), listReader(listReader) {}
+  ObjectReader(ListReader listReader)
+      : kind(LIST), listReader(listReader) {}
 };
 
 // =======================================================================================
@@ -632,7 +634,9 @@ inline T StructBuilder::getDataField(ElementCount offset) const {
 
 template <>
 inline bool StructBuilder::getDataField<bool>(ElementCount offset) const {
-  BitCount boffset = offset * (1 * BITS / ELEMENTS);
+  // This branch should be compiled out whenever this is inlined with a constant offset.
+  BitCount boffset = (offset == 0 * ELEMENTS) ?
+      BitCount(bit0Offset) : offset * (1 * BITS / ELEMENTS);
   byte* b = reinterpret_cast<byte*>(data) + boffset / BITS_PER_BYTE;
   return (*reinterpret_cast<uint8_t*>(b) & (1 << (boffset % BITS_PER_BYTE / BITS))) != 0;
 }
@@ -655,7 +659,9 @@ inline void StructBuilder::setDataField(
 
 template <>
 inline void StructBuilder::setDataField<bool>(ElementCount offset, bool value) const {
-  BitCount boffset = offset * (1 * BITS / ELEMENTS);
+  // This branch should be compiled out whenever this is inlined with a constant offset.
+  BitCount boffset = (offset == 0 * ELEMENTS) ?
+      BitCount(bit0Offset) : offset * (1 * BITS / ELEMENTS);
   byte* b = reinterpret_cast<byte*>(data) + boffset / BITS_PER_BYTE;
   uint bitnum = boffset % BITS_PER_BYTE / BITS;
   *reinterpret_cast<uint8_t*>(b) = (*reinterpret_cast<uint8_t*>(b) & ~(1 << bitnum))
@@ -675,7 +681,7 @@ inline void StructBuilder::setDataField(
 
 template <typename T>
 T StructReader::getDataField(ElementCount offset) const {
-  if ((offset + 1 * ELEMENTS) * capnproto::bytesPerElement<T>() <= dataSize) {
+  if ((offset + 1 * ELEMENTS) * capnproto::bitsPerElement<T>() <= dataSize) {
     return reinterpret_cast<const WireValue<T>*>(data)[offset / ELEMENTS].get();
   } else {
     return static_cast<T>(0);
@@ -685,7 +691,11 @@ T StructReader::getDataField(ElementCount offset) const {
 template <>
 inline bool StructReader::getDataField<bool>(ElementCount offset) const {
   BitCount boffset = offset * (1 * BITS / ELEMENTS);
-  if (boffset < dataSize * BITS_PER_BYTE) {
+  if (boffset < dataSize) {
+    // This branch should be compiled out whenever this is inlined with a constant offset.
+    if (offset == 0 * ELEMENTS) {
+      boffset = bit0Offset;
+    }
     const byte* b = reinterpret_cast<const byte*>(data) + boffset / BITS_PER_BYTE;
     return (*reinterpret_cast<const uint8_t*>(b) & (1 << (boffset % BITS_PER_BYTE / BITS))) != 0;
   } else {
@@ -709,13 +719,20 @@ inline ElementCount ListBuilder::size() { return elementCount; }
 
 template <typename T>
 inline T ListBuilder::getDataElement(ElementCount index) const {
-  return reinterpret_cast<WireValue<T>*>(ptr + index * stepBytes)->get();
+  return reinterpret_cast<WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->get();
+
+  // TODO(soon):  Benchmark this alternate implementation, which I suspect may make better use of
+  //   the x86 SIB byte.  Also use it for all the other getData/setData implementations below, and
+  //   the various non-inline methods that look up pointers.
+  //   Also if using this, consider changing ptr back to void* instead of byte*.
+//  return reinterpret_cast<WireValue<T>*>(ptr)[
+//      index / ELEMENTS * (step / capnproto::bitsPerElement<T>())].get();
 }
 
 template <>
 inline bool ListBuilder::getDataElement<bool>(ElementCount index) const {
   // Ignore stepBytes for bit lists because bit lists cannot be upgraded to struct lists.
-  BitCount bindex = index * (1 * BITS / ELEMENTS);
+  BitCount bindex = index * step;
   byte* b = ptr + bindex / BITS_PER_BYTE;
   return (*reinterpret_cast<uint8_t*>(b) & (1 << (bindex % BITS_PER_BYTE / BITS))) != 0;
 }
@@ -727,7 +744,7 @@ inline Void ListBuilder::getDataElement<Void>(ElementCount index) const {
 
 template <typename T>
 inline void ListBuilder::setDataElement(ElementCount index, typename NoInfer<T>::Type value) const {
-  reinterpret_cast<WireValue<T>*>(ptr + index * stepBytes)->set(value);
+  reinterpret_cast<WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->set(value);
 }
 
 template <>
@@ -749,13 +766,13 @@ inline ElementCount ListReader::size() { return elementCount; }
 
 template <typename T>
 inline T ListReader::getDataElement(ElementCount index) const {
-  return reinterpret_cast<const WireValue<T>*>(ptr + index * stepBytes)->get();
+  return reinterpret_cast<const WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->get();
 }
 
 template <>
 inline bool ListReader::getDataElement<bool>(ElementCount index) const {
   // Ignore stepBytes for bit lists because bit lists cannot be upgraded to struct lists.
-  BitCount bindex = index * (1 * BITS / ELEMENTS);
+  BitCount bindex = index * step;
   const byte* b = ptr + bindex / BITS_PER_BYTE;
   return (*reinterpret_cast<const uint8_t*>(b) & (1 << (bindex % BITS_PER_BYTE / BITS))) != 0;
 }

c++/src/capnproto/message.c++

@@ -53,7 +53,7 @@ internal::StructReader MessageReader::getRootInternal() {
   VALIDATE_INPUT(segment != nullptr &&
       segment->containsInterval(segment->getStartPtr(), segment->getStartPtr() + 1),
       "Message did not contain a root pointer.") {
-    return internal::StructReader::readEmpty();
+    return internal::StructReader();
   }
 
   return internal::StructReader::readRoot(segment->getStartPtr(), segment, options.nestingLimit);

c++/src/capnproto/test-util.c++

@@ -329,7 +329,7 @@ void genericInitListDefaults(Builder builder) {
   auto lists = builder.initLists();
 
   lists.initList0(2);
-  lists.initList1(2);
+  lists.initList1(4);
   lists.initList8(2);
   lists.initList16(2);
   lists.initList32(2);
@@ -340,6 +340,8 @@ void genericInitListDefaults(Builder builder) {
   lists.getList0()[1].setF(Void::VOID);
   lists.getList1()[0].setF(true);
   lists.getList1()[1].setF(false);
+  lists.getList1()[2].setF(true);
+  lists.getList1()[3].setF(true);
   lists.getList8()[0].setF(123u);
   lists.getList8()[1].setF(45u);
   lists.getList16()[0].setF(12345u);
@@ -380,7 +382,7 @@ void genericCheckListDefaults(Reader reader) {
   auto lists = reader.getLists();
 
   ASSERT_EQ(2u, lists.getList0().size());
-  ASSERT_EQ(2u, lists.getList1().size());
+  ASSERT_EQ(4u, lists.getList1().size());
   ASSERT_EQ(2u, lists.getList8().size());
   ASSERT_EQ(2u, lists.getList16().size());
   ASSERT_EQ(2u, lists.getList32().size());
@@ -391,6 +393,8 @@ void genericCheckListDefaults(Reader reader) {
   EXPECT_EQ(Void::VOID, lists.getList0()[1].getF());
   EXPECT_TRUE(lists.getList1()[0].getF());
   EXPECT_FALSE(lists.getList1()[1].getF());
+  EXPECT_TRUE(lists.getList1()[2].getF());
+  EXPECT_TRUE(lists.getList1()[3].getF());
   EXPECT_EQ(123u, lists.getList8()[0].getF());
   EXPECT_EQ(45u, lists.getList8()[1].getF());
   EXPECT_EQ(12345u, lists.getList16()[0].getF());

c++/src/capnproto/test.capnp

@@ -308,10 +308,16 @@ struct TestLists {
   structListList @9 :List(List(TestAllTypes));
 }
 
+struct TestFieldZeroIsBit {
+  bit @0 :Bool;
+  secondBit @1 :Bool = true;
+  thirdField @2 :UInt8 = 123;
+}
+
 struct TestListDefaults {
   lists @0 :TestLists = (
       list0  = [(f = void), (f = void)],
-      list1  = [(f = true), (f = false)],
+      list1  = [(f = true), (f = false), (f = true), (f = true)],
       list8  = [(f = 123), (f = 45)],
       list16 = [(f = 12345), (f = 6789)],
       list32 = [(f = 123456789), (f = 234567890)],

c++/src/capnproto/type-safety.h

@@ -462,6 +462,19 @@ struct Id {
 // =======================================================================================
 // Units
 
+template <typename T> constexpr bool isIntegral() { return false; }
+template <> constexpr bool isIntegral<char>() { return true; }
+template <> constexpr bool isIntegral<signed char>() { return true; }
+template <> constexpr bool isIntegral<short>() { return true; }
+template <> constexpr bool isIntegral<int>() { return true; }
+template <> constexpr bool isIntegral<long>() { return true; }
+template <> constexpr bool isIntegral<long long>() { return true; }
+template <> constexpr bool isIntegral<unsigned char>() { return true; }
+template <> constexpr bool isIntegral<unsigned short>() { return true; }
+template <> constexpr bool isIntegral<unsigned int>() { return true; }
+template <> constexpr bool isIntegral<unsigned long>() { return true; }
+template <> constexpr bool isIntegral<unsigned long long>() { return true; }
+
 template <typename Number, typename Unit1, typename Unit2>
 class UnitRatio {
   // A multiplier used to convert Quantities of one unit to Quantities of another unit.  See
@@ -470,6 +483,8 @@ class UnitRatio {
   // Construct this type by dividing one Quantity by another of a different unit.  Use this type
   // by multiplying it by a Quantity, or dividing a Quantity by it.
 
+  static_assert(isIntegral<Number>(), "Underlying type for UnitRatio must be integer.");
+
 public:
   inline UnitRatio() {}
 
@@ -529,12 +544,14 @@ private:
   friend class UnitRatio;
 
   template <typename N1, typename N2, typename U1, typename U2>
-  friend inline constexpr decltype(N1(1) * N2(1)) operator*(N1, UnitRatio<N2, U1, U2>);
+  friend inline constexpr UnitRatio<decltype(N1(1) * N2(1)), U1, U2>
+      operator*(N1, UnitRatio<N2, U1, U2>);
 };
 
 template <typename N1, typename N2, typename U1, typename U2>
-inline constexpr decltype(N1(1) * N2(1)) operator*(N1 n, UnitRatio<N2, U1, U2> r) {
-  return n * r.unit1PerUnit2;
+inline constexpr UnitRatio<decltype(N1(1) * N2(1)), U1, U2>
+    operator*(N1 n, UnitRatio<N2, U1, U2> r) {
+  return UnitRatio<decltype(N1(1) * N2(1)), U1, U2>(n * r.unit1PerUnit2);
 }
 
 template <typename Number, typename Unit>
@@ -581,6 +598,8 @@ class Quantity {
   //     waitFor(3 * MINUTES);
   //   }
 
+  static_assert(isIntegral<Number>(), "Underlying type for Quantity must be integer.");
+
 public:
   inline constexpr Quantity() {}
 
@@ -605,11 +624,13 @@ public:
   template <typename OtherNumber>
   inline constexpr Quantity<decltype(Number(1) * OtherNumber(1)), Unit>
       operator*(OtherNumber other) const {
+    static_assert(isIntegral<OtherNumber>(), "Multiplied Quantity by non-integer.");
     return Quantity<decltype(Number(1) * other), Unit>(value * other);
   }
   template <typename OtherNumber>
   inline constexpr Quantity<decltype(Number(1) / OtherNumber(1)), Unit>
       operator/(OtherNumber other) const {
+    static_assert(isIntegral<OtherNumber>(), "Divided Quantity by non-integer.");
     return Quantity<decltype(Number(1) / other), Unit>(value / other);
   }
   template <typename OtherNumber>

compiler/src/Compiler.hs

@@ -708,8 +708,7 @@ packUnion :: UnionDesc -> PackingState -> Map.Map Integer UnionPackingState
 packUnion _ state unionState = (DataOffset Size16 offset, newState, unionState) where
     (offset, newState) = packData Size16 state
 
-stripHolesFromFirstWord Size1 _ = error "can't get this far"
-stripHolesFromFirstWord Size8 _ = Size8  -- Don't reduce to less than a byte.
+stripHolesFromFirstWord Size1 _ = Size1  -- Stop at a bit.
 stripHolesFromFirstWord size holes = let
     nextSize = pred size
     in case Map.lookup nextSize holes of
@@ -744,6 +743,7 @@ enforceFixed Nothing sizes = return sizes
 enforceFixed (Just (Located pos (requestedDataSize, requestedPointerCount)))
         (actualDataSize, actualPointerCount) = do
     validatedRequestedDataSize <- case requestedDataSize of
+        1 -> return DataSection1
         8 -> return DataSection8
         16 -> return DataSection16
         32 -> return DataSection32

compiler/src/CxxGenerator.hs

@@ -382,7 +382,7 @@ structContext parent desc = mkStrContext context where
     context "structReferenceCount" = MuVariable $ structPointerCount desc
     context "structPreferredListEncoding" = case (structDataSize desc, structPointerCount desc) of
         (DataSectionWords 0, 0) -> MuVariable "VOID"
-        (DataSection1, 0) -> MuVariable "BYTE"
+        (DataSection1, 0) -> MuVariable "BIT"
         (DataSection8, 0) -> MuVariable "BYTE"
         (DataSection16, 0) -> MuVariable "TWO_BYTES"
         (DataSection32, 0) -> MuVariable "FOUR_BYTES"

compiler/src/Semantics.hs

@@ -318,9 +318,9 @@ fieldSize (InlineStructType StructDesc { structDataSize = ds, structPointerCount
 fieldSize (InterfaceType _) = SizeReference
 fieldSize (ListType _) = SizeReference
 fieldSize (InlineListType element size) = let
-    -- We intentionally do not allow single-bit lists because most CPUs cannot address bits.
     minDataSectionForBits bits
         | bits <= 0 = DataSectionWords 0
+        | bits <= 1 = DataSection1
         | bits <= 8 = DataSection8
         | bits <= 16 = DataSection16
         | bits <= 32 = DataSection32

doc/encoding.md

@@ -114,11 +114,12 @@ A list value is encoded as a pointer to a flat array of values.
 The pointed-to values are tightly-packed.  In particular, `Bool`s are packed bit-by-bit in
 little-endian order (the first bit is the least-significant bit of the first byte).
 
-Lists of structs use the smallest element size in which the struct can fit, except that single-bit
-structs are not allowed.  So, a list of structs that each contain two `UInt8` fields and nothing
-else could be encoded with C = 3 (2-byte elements).  A list of structs that each contain a single
-`Text` field would be encoded as C = 6 (pointer elements).  A list structs which are each more than
-one word in size must be be encoded using C = 7 (composite).
+Lists of structs use the smallest element size in which the struct can fit.  So, a
+list of structs that each contain two `UInt8` fields and nothing else could be encoded with C = 3
+(2-byte elements).  A list of structs that each contain a single `Text` field would be encoded as
+C = 6 (pointer elements).  A list of structs that each contain a single `Bool` field would be
+encoded using C = 1 (1-bit elements).  A list structs which are each more than one word in size
+must be be encoded using C = 7 (composite).
 
 When C = 7, the elements of the list are fixed-width composite values -- usually, structs.  In
 this case, the list content is prefixed by a "tag" word that describes each individual element.
@@ -136,7 +137,7 @@ elements being fixed-size lists rather than structs.  In this case, the tag woul
 pointer rather than a struct pointer.  As of this writing, no such feature has been implemented.
 
 Notice that because a small struct is encoded as if it were a primitive value, this means that
-if you have a field of type `List(T)` where `T` is a primitive or blob type (other than `Bool`), it
+if you have a field of type `List(T)` where `T` is a primitive or blob type, it
 is possible to change that field to `List(U)` where `U` is a struct whose `@0` field has type `T`,
 without breaking backwards-compatibility.  This comes in handy when you discover too late that you
 need to associate some extra data with each value in a primitive list -- instead of using parallel

doc/language.md

@@ -489,12 +489,11 @@ A protocol can be changed in the following ways without breaking backwards-compa
   parameter list and must have default values.
 * Any symbolic name can be changed, as long as the ordinal numbers stay the same.
 * Types definitions can be moved to different scopes.
-* A field of type `List(T)`, where `T` is a primitive type (except `Bool`), non-inline blob, or
+* A field of type `List(T)`, where `T` is a primitive type, non-inline blob, or
   non-inline list, may be changed to type `List(U)`, where `U` is a struct type whose `@0` field is
   of type `T`.  This rule is useful when you realize too late that you need to attach some extra
   data to each element of your list.  Without this rule, you would be stuck defining parallel
-  lists, which are ugly and error-prone.  (`List(Bool)` does not support this transformation
-  because it would be difficult to implement given that booleans are packed 8 to the byte.)
+  lists, which are ugly and error-prone.
 * A struct that is not already `fixed` can be made `fixed`.  However, once a struct is declared
   `fixed`, the declaration cannot be removed or changed, as this would change the layout of `Inline`
   uses of the struct.