Add a way to concatenate lists without losing data when one or more of the lists…

Add a way to concatenate lists without losing data when one or more of the lists was written using a newer version of the protocol.

c++/src/capnp/layout.c++

@@ -2869,6 +2869,91 @@ OrphanBuilder OrphanBuilder::copy(
 }
 #endif  // !CAPNP_LITE
 
+OrphanBuilder OrphanBuilder::concat(
+    BuilderArena* arena, CapTableBuilder* capTable,
+    ElementSize elementSize, StructSize structSize,
+    kj::ArrayPtr<const ListReader> lists) {
+  KJ_REQUIRE(lists.size() > 0, "Can't concat empty list ");
+
+  // Find the overall element count and size.
+  ElementCount elementCount = 0 * ELEMENTS;
+  for (auto& list: lists) {
+    elementCount += list.elementCount;
+    if (list.elementSize != elementSize) {
+      // If element sizes don't all match, upgrade to struct list.
+      KJ_REQUIRE(list.elementSize != ElementSize::BIT && elementSize != ElementSize::BIT,
+                 "can't upgrade bit lists to struct lists");
+      elementSize = ElementSize::INLINE_COMPOSITE;
+    }
+    structSize.data = kj::max(structSize.data,
+        WireHelpers::roundBitsUpToWords(list.structDataSize));
+    structSize.pointers = kj::max(structSize.pointers, list.structPointerCount);
+  }
+
+  // Allocate the list.
+  OrphanBuilder result;
+  ListBuilder builder = (elementSize == ElementSize::INLINE_COMPOSITE)
+      ? WireHelpers::initStructListPointer(
+          result.tagAsPtr(), nullptr, capTable, elementCount, structSize, arena)
+      : WireHelpers::initListPointer(
+          result.tagAsPtr(), nullptr, capTable, elementCount, elementSize, arena);
+
+  // Copy elements.
+  switch (elementSize) {
+    case ElementSize::INLINE_COMPOSITE: {
+      ElementCount pos = 0 * ELEMENTS;
+      for (auto& list: lists) {
+        for (ElementCount i = 0 * ELEMENTS; i < list.size(); i += 1 * ELEMENTS) {
+          builder.getStructElement(pos).copyContentFrom(list.getStructElement(i));
+          pos += 1 * ELEMENTS;
+        }
+      }
+      break;
+    }
+    case ElementSize::POINTER: {
+      ElementCount pos = 0 * ELEMENTS;
+      for (auto& list: lists) {
+        for (ElementCount i = 0 * ELEMENTS; i < list.size(); i += 1 * ELEMENTS) {
+          builder.getPointerElement(pos).copyFrom(list.getPointerElement(i));
+          pos += 1 * ELEMENTS;
+        }
+      }
+      break;
+    }
+    case ElementSize::BIT: {
+      // It's difficult to memcpy() bits since a list could start or end mid-byte. For now we
+      // do a slow, naive loop. Probably no one will ever care.
+      ElementCount pos = 0 * ELEMENTS;
+      for (auto& list: lists) {
+        for (ElementCount i = 0 * ELEMENTS; i < list.size(); i += 1 * ELEMENTS) {
+          builder.setDataElement<bool>(pos, list.getDataElement<bool>(i));
+          pos += 1 * ELEMENTS;
+        }
+      }
+      break;
+    }
+    default: {
+      // We know all the inputs had identical size because otherwise we would have chosen
+      // INLINE_COMPOSITE. Therefore, we can safely use memcpy() here instead of copying each
+      // element manually.
+      byte* target = builder.ptr;
+      auto step = builder.step / BITS_PER_BYTE;
+      for (auto& list: lists) {
+        auto count = step * list.size();
+        memcpy(target, list.ptr, count / BYTES);
+        target += count / BYTES;
+      }
+      break;
+    }
+  }
+
+  // Return orphan.
+  result.segment = builder.segment;
+  result.capTable = capTable;
+  result.location = builder.getLocation();
+  return result;
+}
+
 OrphanBuilder OrphanBuilder::referenceExternalData(BuilderArena* arena, Data::Reader data) {
   KJ_REQUIRE(reinterpret_cast<uintptr_t>(data.begin()) % sizeof(void*) == 0,
              "Cannot referenceExternalData() that is not aligned.");

c++/src/capnp/layout.h

@@ -183,6 +183,25 @@ inline constexpr StructSize structSize() {
   return StructSize(CapnpPrivate::dataWordSize * WORDS, CapnpPrivate::pointerCount * POINTERS);
 }
 
+template <typename T, typename CapnpPrivate = typename T::_capnpPrivate,
+          typename = kj::EnableIf<CAPNP_KIND(T) == Kind::STRUCT>>
+inline constexpr StructSize minStructSizeForElement() {
+  // If T is a struct, return its struct size. Otherwise return the minimum struct size big enough
+  // to hold a T.
+
+  return StructSize(CapnpPrivate::dataWordSize * WORDS, CapnpPrivate::pointerCount * POINTERS);
+}
+
+template <typename T, typename = kj::EnableIf<CAPNP_KIND(T) != Kind::STRUCT>>
+inline constexpr StructSize minStructSizeForElement() {
+  // If T is a struct, return its struct size. Otherwise return the minimum struct size big enough
+  // to hold a T.
+
+  return StructSize(
+      dataBitsPerElement(elementSizeForType<T>()) * ELEMENTS > 0 * BITS ? 1 * WORDS : 0 * WORDS,
+      pointersPerElement(elementSizeForType<T>()) * ELEMENTS);
+}
+
 // -------------------------------------------------------------------
 // Masking of default values
 
@@ -804,6 +823,10 @@ public:
                             kj::Own<ClientHook> copyFrom);
 #endif  // !CAPNP_LITE
 
+  static OrphanBuilder concat(BuilderArena* arena, CapTableBuilder* capTable,
+                              ElementSize expectedElementSize, StructSize expectedStructSize,
+                              kj::ArrayPtr<const ListReader> lists);
+
   static OrphanBuilder referenceExternalData(BuilderArena* arena, Data::Reader data);
 
   OrphanBuilder& operator=(const OrphanBuilder& other) = delete;

c++/src/capnp/list.h

@@ -272,6 +272,10 @@ struct List<T, Kind::STRUCT> {
       // If the source struct is larger than the target struct -- say, because the source was built
       // using a newer version of the schema that has additional fields -- it will be truncated,
       // losing data.
+      //
+      // Note: If you are trying to concatenate some lists, use Orphanage::newOrphanConcat() to
+      //   do it without losing any data in case the source lists come from a newer version of the
+      //   protocol. (Plus, it's easier to use anyhow.)
 
       KJ_IREQUIRE(index < size());
       builder.getStructElement(index * ELEMENTS).copyContentFrom(reader._reader);

c++/src/capnp/orphan-test.c++

@@ -1540,6 +1540,179 @@ TEST(Orphans, ExtendStructListFromEmpty) {
   }
 }
 
+template <typename ListBuilder>
+void initList(ListBuilder builder,
+    std::initializer_list<ReaderFor<ListElementType<FromBuilder<ListBuilder>>>> values) {
+  KJ_ASSERT(builder.size() == values.size());
+  size_t i = 0;
+  for (auto& value: values) {
+    builder.set(i++, value);
+  }
+}
+
+TEST(Orphans, ConcatenatePrimitiveLists) {
+  MallocMessageBuilder message;
+  auto orphanage = message.getOrphanage();
+
+  auto list1 = orphanage.newOrphan<List<uint32_t>>(3);
+  initList(list1.get(), {12, 34, 56});
+
+  auto list2 = orphanage.newOrphan<List<uint32_t>>(2);
+  initList(list2.get(), {78, 90});
+
+  auto list3 = orphanage.newOrphan<List<uint32_t>>(4);
+  initList(list3.get(), {23, 45, 67, 89});
+
+  List<uint32_t>::Reader lists[] = { list1.getReader(), list2.getReader(), list3.getReader() };
+  kj::ArrayPtr<List<uint32_t>::Reader> array = lists;
+
+  auto cat = message.getOrphanage().newOrphanConcat(array);
+
+  checkList(cat.getReader(), {12, 34, 56, 78, 90, 23, 45, 67, 89});
+}
+
+TEST(Orphans, ConcatenateBitLists) {
+  MallocMessageBuilder message;
+  auto orphanage = message.getOrphanage();
+
+  auto list1 = orphanage.newOrphan<List<bool>>(3);
+  initList(list1.get(), {true, true, false});
+
+  auto list2 = orphanage.newOrphan<List<bool>>(2);
+  initList(list2.get(), {false, true});
+
+  auto list3 = orphanage.newOrphan<List<bool>>(4);
+  initList(list3.get(), {false, false, true, false});
+
+  List<bool>::Reader lists[] = { list1.getReader(), list2.getReader(), list3.getReader() };
+  kj::ArrayPtr<List<bool>::Reader> array = lists;
+
+  auto cat = message.getOrphanage().newOrphanConcat(array);
+
+  checkList(cat.getReader(), {true, true, false, false, true, false, false, true, false});
+}
+
+TEST(Orphans, ConcatenatePointerLists) {
+  MallocMessageBuilder message;
+  auto orphanage = message.getOrphanage();
+
+  auto list1 = orphanage.newOrphan<List<Text>>(3);
+  initList(list1.get(), {"foo", "bar", "baz"});
+
+  auto list2 = orphanage.newOrphan<List<Text>>(2);
+  initList(list2.get(), {"qux", "corge"});
+
+  auto list3 = orphanage.newOrphan<List<Text>>(4);
+  initList(list3.get(), {"grault", "garply", "waldo", "fred"});
+
+  List<Text>::Reader lists[] = { list1.getReader(), list2.getReader(), list3.getReader() };
+  kj::ArrayPtr<List<Text>::Reader> array = lists;
+
+  auto cat = message.getOrphanage().newOrphanConcat(array);
+
+  checkList(cat.getReader(), {
+      "foo", "bar", "baz", "qux", "corge", "grault", "garply", "waldo", "fred"});
+}
+
+TEST(Orphans, ConcatenateStructLists) {
+  // In this test, we not only concatenate two struct lists, but we concatenate in a list that
+  // contains a newer-than-expected version of the struct with extra fields, in order to verify
+  // that the new fields aren't lost.
+
+  MallocMessageBuilder message;
+  auto orphanage = message.getOrphanage();
+
+  auto orphan1 = orphanage.newOrphan<List<test::TestOldVersion>>(2);
+  auto list1 = orphan1.get();
+  list1[0].setOld1(12);
+  list1[0].setOld2("foo");
+  list1[1].setOld1(34);
+  list1[1].setOld2("bar");
+
+  auto orphan2 = orphanage.newOrphan<test::TestAnyPointer>();
+  auto list2 = orphan2.get().getAnyPointerField().initAs<List<test::TestNewVersion>>(2);
+  list2[0].setOld1(56);
+  list2[0].setOld2("baz");
+  list2[0].setNew1(123);
+  list2[0].setNew2("corge");
+  list2[1].setOld1(78);
+  list2[1].setOld2("qux");
+  list2[1].setNew1(456);
+  list2[1].setNew2("grault");
+
+  List<test::TestOldVersion>::Reader lists[] = {
+    orphan1.getReader(),
+    orphan2.getReader().getAnyPointerField().getAs<List<test::TestOldVersion>>()
+  };
+  kj::ArrayPtr<List<test::TestOldVersion>::Reader> array = lists;
+
+  auto orphan3 = orphanage.newOrphan<test::TestAnyPointer>();
+  orphan3.get().getAnyPointerField().adopt(message.getOrphanage().newOrphanConcat(array));
+
+  auto cat = orphan3.getReader().getAnyPointerField().getAs<List<test::TestNewVersion>>();
+  ASSERT_EQ(4, cat.size());
+
+  EXPECT_EQ(12, cat[0].getOld1());
+  EXPECT_EQ("foo", cat[0].getOld2());
+  EXPECT_EQ(987, cat[0].getNew1());
+  EXPECT_FALSE(cat[0].hasNew2());
+
+  EXPECT_EQ(34, cat[1].getOld1());
+  EXPECT_EQ("bar", cat[1].getOld2());
+  EXPECT_EQ(987, cat[1].getNew1());
+  EXPECT_FALSE(cat[1].hasNew2());
+
+  EXPECT_EQ(56, cat[2].getOld1());
+  EXPECT_EQ("baz", cat[2].getOld2());
+  EXPECT_EQ(123, cat[2].getNew1());
+  EXPECT_EQ("corge", cat[2].getNew2());
+
+  EXPECT_EQ(78, cat[3].getOld1());
+  EXPECT_EQ("qux", cat[3].getOld2());
+  EXPECT_EQ(456, cat[3].getNew1());
+  EXPECT_EQ("grault", cat[3].getNew2());
+}
+
+TEST(Orphans, ConcatenateStructListsUpgradeFromPrimitive) {
+  // Like above, but we're "upgrading" a primitive list to a struct list.
+
+  MallocMessageBuilder message;
+  auto orphanage = message.getOrphanage();
+
+  auto orphan1 = orphanage.newOrphan<List<test::TestOldVersion>>(2);
+  auto list1 = orphan1.get();
+  list1[0].setOld1(12);
+  list1[0].setOld2("foo");
+  list1[1].setOld1(34);
+  list1[1].setOld2("bar");
+
+  auto orphan2 = orphanage.newOrphan<test::TestAnyPointer>();
+  auto list2 = orphan2.get().getAnyPointerField().initAs<List<uint64_t>>(2);
+  initList(list2, {12345, 67890});
+
+  List<test::TestOldVersion>::Reader lists[] = {
+    orphan1.getReader(),
+    orphan2.getReader().getAnyPointerField().getAs<List<test::TestOldVersion>>()
+  };
+  kj::ArrayPtr<List<test::TestOldVersion>::Reader> array = lists;
+
+  auto orphan3 = message.getOrphanage().newOrphanConcat(array);
+  auto cat = orphan3.getReader();
+  ASSERT_EQ(4, cat.size());
+
+  EXPECT_EQ(12, cat[0].getOld1());
+  EXPECT_EQ("foo", cat[0].getOld2());
+
+  EXPECT_EQ(34, cat[1].getOld1());
+  EXPECT_EQ("bar", cat[1].getOld2());
+
+  EXPECT_EQ(12345, cat[2].getOld1());
+  EXPECT_FALSE(cat[2].hasOld2());
+
+  EXPECT_EQ(67890, cat[3].getOld1());
+  EXPECT_FALSE(cat[3].hasOld2());
+}
+
 }  // namespace
 }  // namespace _ (private)
 }  // namespace capnp

c++/src/capnp/orphan.h

@@ -142,6 +142,19 @@ public:
   // Allocate a new orphaned object (struct, list, or blob) and initialize it as a copy of the
   // given object.
 
+  template <typename T>
+  Orphan<List<ListElementType<FromReader<T>>>> newOrphanConcat(kj::ArrayPtr<T> lists) const;
+  template <typename T>
+  Orphan<List<ListElementType<FromReader<T>>>> newOrphanConcat(kj::ArrayPtr<const T> lists) const;
+  // Given an array of List readers, copy and concatenate the lists, creating a new Orphan.
+  //
+  // Note that compared to allocating the list yourself and using `setWithCaveats()` to set each
+  // item, this method avoids the "caveats": the new list will be allocated with the element size
+  // being the maximum of that from all the input lists. This is particularly important when
+  // concatenating struct lists: if the lists were created using a newer version of the protocol
+  // in which some new fields had been added to the struct, using `setWithCaveats()` would
+  // truncate off those new fields.
+
   Orphan<Data> referenceExternalData(Data::Reader data) const;
   // Creates an Orphan<Data> that points at an existing region of memory (e.g. from another message)
   // without copying it.  There are some SEVERE restrictions on how this can be used:
@@ -404,6 +417,26 @@ inline Orphan<FromReader<Reader>> Orphanage::newOrphanCopy(Reader& copyFrom) con
   return newOrphanCopy(kj::implicitCast<const Reader&>(copyFrom));
 }
 
+template <typename T>
+inline Orphan<List<ListElementType<FromReader<T>>>>
+Orphanage::newOrphanConcat(kj::ArrayPtr<T> lists) const {
+  return newOrphanConcat(kj::implicitCast<kj::ArrayPtr<const T>>(lists));
+}
+template <typename T>
+inline Orphan<List<ListElementType<FromReader<T>>>>
+Orphanage::newOrphanConcat(kj::ArrayPtr<const T> lists) const {
+  // Optimization / simplification: Rely on List<T>::Reader containing nothing except a
+  // _::ListReader.
+  static_assert(sizeof(T) == sizeof(_::ListReader), "lists are not bare readers?");
+  kj::ArrayPtr<const _::ListReader> raw(
+      reinterpret_cast<const _::ListReader*>(lists.begin()), lists.size());
+  typedef ListElementType<FromReader<T>> Element;
+  return Orphan<List<Element>>(
+      _::OrphanBuilder::concat(arena, capTable,
+          _::elementSizeForType<Element>(),
+          _::minStructSizeForElement<Element>(), raw));
+}
+
 inline Orphan<Data> Orphanage::referenceExternalData(Data::Reader data) const {
   return Orphan<Data>(_::OrphanBuilder::referenceExternalData(arena, data));
 }