Fully implement Orphan::truncate() and allow it to be used even to extend lists.

c++/src/capnp/arena.h

@@ -177,6 +177,11 @@ public:
   // If `from` points just past the current end of the segment, then move the end back to `to`.
   // Otherwise, do nothing.
 
+  inline bool tryExtend(word* from, word* to);
+  // If `from` points just past the current end of the segment, and `to` is within the segment
+  // boundaries, then move the end up to `to` and return true. Otherwise, do nothing and return
+  // false.
+
 private:
   word* pos;
   // Pointer to a pointer to the current end point of the segment, i.e. the location where the
@@ -445,6 +450,16 @@ inline void SegmentBuilder::tryTruncate(word* from, word* to) {
   if (pos == from) pos = to;
 }
 
+inline bool SegmentBuilder::tryExtend(word* from, word* to) {
+  // Careful about overflow.
+  if (pos == from && to <= ptr.end() && to >= from) {
+    pos = to;
+    return true;
+  } else {
+    return false;
+  }
+}
+
 }  // namespace _ (private)
 }  // namespace capnp
 

c++/src/capnp/layout.c++

@@ -2294,6 +2294,7 @@ void PointerBuilder::transferFrom(PointerBuilder other) {
     memset(pointer, 0, sizeof(*pointer));
   }
   WireHelpers::transferPointer(segment, pointer, other.segment, other.pointer);
+  memset(other.pointer, 0, sizeof(*other.pointer));
 }
 
 void PointerBuilder::copyFrom(PointerReader other) {
@@ -2613,10 +2614,10 @@ kj::ArrayPtr<const byte> ListReader::asRawBytes() {
     return kj::ArrayPtr<const byte>();
   }
 
-  return kj::ArrayPtr<const byte>(reinterpret_cast<const byte*>(ptr), structDataSize * elementCount / ELEMENTS);
+  return kj::ArrayPtr<const byte>(reinterpret_cast<const byte*>(ptr),
+      WireHelpers::roundBitsUpToBytes(elementCount * (structDataSize / ELEMENTS)) / BYTES);
 }
 
-
 StructReader ListReader::getStructElement(ElementCount index) const {
   KJ_REQUIRE(nestingLimit > 0,
              "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
@@ -2851,26 +2852,123 @@ void OrphanBuilder::truncate(ElementCount size, bool isText) {
     return;
   }
 
-  // TODO(someday): Implement truncation of all sizes.
-  KJ_ASSERT(ref->listRef.elementSize() == ElementSize::BYTE,
-            "Not implemented: truncate non-blob.");
+  ElementSize elementSize = ref->listRef.elementSize();
 
-  auto oldSize = ref->listRef.elementCount();
-  KJ_REQUIRE(size <= oldSize, "Truncate size must be smaller than existing size.") {
-    return;
-  }
+  if (elementSize == ElementSize::INLINE_COMPOSITE) {
+    WordCount oldWordCount = ref->listRef.inlineCompositeWordCount();
 
-  ref->listRef.set(ref->listRef.elementSize(), size);
+    WirePointer* tag = reinterpret_cast<WirePointer*>(target);
+    ++target;
+    KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
+               "INLINE_COMPOSITE lists of non-STRUCT type are not supported.") {
+      return;
+    }
+    StructSize structSize(tag->structRef.dataSize.get(), tag->structRef.ptrCount.get());
+    WordCount elementWordCount = structSize.total();
+
+    ElementCount oldSize = tag->inlineCompositeListElementCount();
+    word* newEndWord = target + size * (elementWordCount / ELEMENTS);
+    word* oldEndWord = target + oldWordCount;
 
-  byte* begin = reinterpret_cast<byte*>(target);
-  byte* truncPoint = begin + size * (1 * BYTES / ELEMENTS);
-  byte* end = begin + oldSize * (1 * BYTES / ELEMENTS);
-  memset(truncPoint - isText, 0, end - truncPoint + isText);
+    if (size <= oldSize) {
+      // Zero the trailing elements.
+      for (uint i = size / ELEMENTS; i < oldSize / ELEMENTS; i++) {
+        WireHelpers::zeroObject(segment, tag, target + i * elementWordCount);
+      }
+      ref->listRef.setInlineComposite(size * (elementWordCount / ELEMENTS));
+      tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
+      segment->tryTruncate(oldEndWord, newEndWord);
+    } else if (newEndWord <= oldEndWord) {
+      // Apparently the old list was over-allecated? The word count is more than needed to store
+      // the elements. This is "valid" but shouldn't happen in practice unless someone is toying
+      // with us.
+      word* expectedEnd = target + oldSize * (elementWordCount / ELEMENTS);
+      KJ_ASSERT(newEndWord >= expectedEnd);
+      memset(expectedEnd, 0, (newEndWord - expectedEnd) * sizeof(word));
+      tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
+    } else {
+      if (segment->tryExtend(oldEndWord, newEndWord)) {
+        // Done in-place. Nothing else to do now; the new memory is already zero'd.
+        ref->listRef.setInlineComposite(size * (elementWordCount / ELEMENTS));
+        tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
+      } else {
+        // Need to re-allocate and transfer.
+        StructSize structSize(tag->structRef.dataSize.get(), tag->structRef.ptrCount.get());
+        OrphanBuilder replacement = initStructList(segment->getArena(), size, structSize);
+
+        ListBuilder newList = replacement.asStructList(structSize);
+        word* element = target;
+        for (uint i = 0; i < oldSize / ELEMENTS; i++) {
+          newList.getStructElement(i * ELEMENTS).transferContentFrom(
+              StructBuilder(segment, element,
+                            reinterpret_cast<WirePointer*>(element + structSize.data),
+                            structSize.data * BITS_PER_WORD, structSize.pointers));
+          element += elementWordCount;
+        }
 
-  word* truncWord = target + WireHelpers::roundBytesUpToWords(size * (1 * BYTES / ELEMENTS));
-  word* endWord = target + WireHelpers::roundBytesUpToWords(oldSize * (1 * BYTES / ELEMENTS));
+        *this = kj::mv(replacement);
+      }
+    }
+  } else if (elementSize == ElementSize::POINTER) {
+    auto oldSize = ref->listRef.elementCount();
+    word* newEndWord = target + size * (POINTER_SIZE_IN_WORDS / ELEMENTS);
+    word* oldEndWord = target + oldSize * (POINTER_SIZE_IN_WORDS / ELEMENTS);
 
-  segment->tryTruncate(endWord, truncWord);
+    if (size <= oldSize) {
+      // Zero the trailing elements.
+      for (WirePointer* element = reinterpret_cast<WirePointer*>(newEndWord);
+           element < reinterpret_cast<WirePointer*>(oldEndWord); ++element) {
+        WireHelpers::zeroPointerAndFars(segment, element);
+      }
+      ref->listRef.set(ElementSize::POINTER, size);
+      segment->tryTruncate(oldEndWord, newEndWord);
+    } else {
+      if (segment->tryExtend(oldEndWord, newEndWord)) {
+        // Done in-place. Nothing else to do now; the new memory is already zero'd.
+        ref->listRef.set(ElementSize::POINTER, size);
+      } else {
+        // Need to re-allocate and transfer.
+        OrphanBuilder replacement = initList(segment->getArena(), size, ElementSize::POINTER);
+        ListBuilder newList = replacement.asList(ElementSize::POINTER);
+        WirePointer* oldPointers = reinterpret_cast<WirePointer*>(target);
+        for (uint i = 0; i < oldSize / ELEMENTS; i++) {
+          newList.getPointerElement(i * ELEMENTS).transferFrom(
+              PointerBuilder(segment, oldPointers + i));
+        }
+        *this = kj::mv(replacement);
+      }
+    }
+  } else {
+    auto oldSize = ref->listRef.elementCount();
+    auto step = dataBitsPerElement(elementSize);
+    word* newEndWord = target + WireHelpers::roundBitsUpToWords(size * step);
+    word* oldEndWord = target + WireHelpers::roundBitsUpToWords(oldSize * step);
+
+    if (size <= oldSize) {
+      // When truncating text, we want to set the null terminator as well, so we'll do our zeroing
+      // at the byte level.
+      byte* begin = reinterpret_cast<byte*>(target);
+      byte* newEndByte = begin + WireHelpers::roundBitsUpToBytes(size * step) - isText;
+      byte* oldEndByte = reinterpret_cast<byte*>(oldEndWord);
+
+      memset(newEndByte, 0, oldEndByte - newEndByte);
+      ref->listRef.set(elementSize, size);
+      segment->tryTruncate(oldEndWord, newEndWord);
+    } else {
+      // We're trying to extend, not truncate.
+      if (segment->tryExtend(oldEndWord, newEndWord)) {
+        // Done in-place. Nothing else to do now; the memory is already zero'd.
+        ref->listRef.set(elementSize, size);
+      } else {
+        // Need to re-allocate and transfer.
+        OrphanBuilder replacement = initList(segment->getArena(), size, elementSize);
+        ListBuilder newList = replacement.asList(elementSize);
+        auto words = WireHelpers::roundBitsUpToWords(dataBitsPerElement(elementSize) * oldSize);
+        memcpy(newList.ptr, target, words * BYTES_PER_WORD / BYTES);
+        *this = kj::mv(replacement);
+      }
+    }
+  }
 }
 
 void OrphanBuilder::euthanize() {

c++/src/capnp/layout.h

@@ -334,6 +334,7 @@ private:
 
   friend class StructBuilder;
   friend class ListBuilder;
+  friend class OrphanBuilder;
 };
 
 class PointerReader {

c++/src/capnp/list.h

@@ -386,7 +386,7 @@ struct List<List<T>, Kind::LIST> {
     }
     inline void adopt(uint index, Orphan<T>&& value) {
       KJ_IREQUIRE(index < size());
-      builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value));
+      builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value.builder));
     }
     inline Orphan<T> disown(uint index) {
       KJ_IREQUIRE(index < size());
@@ -484,7 +484,7 @@ struct List<T, Kind::BLOB> {
     }
     inline void adopt(uint index, Orphan<T>&& value) {
       KJ_IREQUIRE(index < size());
-      builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value));
+      builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value.builder));
     }
     inline Orphan<T> disown(uint index) {
       KJ_IREQUIRE(index < size());

c++/src/capnp/orphan.h

@@ -67,14 +67,24 @@ public:
   inline bool operator!=(decltype(nullptr)) const { return builder != nullptr; }
 
   inline void truncate(uint size);
-  // Truncate the object (which must be a list or a blob) down to the given size. The object's
-  // current size must be larger than this. The object stays in its current position. If the object
-  // is the last object in its segment (which is always true if the object is the last thing that
-  // was allocated in the message) then the truncated space can be reclaimed. Otherwise, the space
-  // is zero'd out but otherwise lost, like an abandoned orphan.
+  // Resize an object (which must be a list or a blob) to the given size.
   //
-  // Any existing readers or builders pointing at the object are invalidated by this call.  You
-  // must call `get()` or `getReader()` again to get the new, valid pointer.
+  // If the new size is less than the original, the remaining elements will be discarded. The
+  // list is never moved in this case. If the list happens to be located at the end of its segment
+  // (which is always true if the list was the last thing allocated), the removed memory will be
+  // reclaimed (reducing the messag size), otherwise it is simply zeroed. The reclaiming behavior
+  // is particularly useful for allocating buffer space when you aren't sure how much space you
+  // actually need: you can pre-allocate, say, a 4k byte array, read() from a file into it, and
+  // then truncate it back to the amount of space actually used.
+  //
+  // If the new size is greater than the original, the list is extended with default values. If
+  // the list is the last object in its segment *and* there is enough space left in the segment to
+  // extend it to cover the new values, then the list is extended in-place. Otherwise, it must be
+  // moved to a new location, leaving a zero'd hole in the previous space that won't be filled.
+  // This copy is shallow; sub-objects will simply be reparented, not copied.
+  //
+  // Any existing readers or builders pointing at the object are invalidated by this call (even if
+  // it doesn't move). You must call `get()` or `getReader()` again to get the new, valid pointer.
 
 private:
   _::OrphanBuilder builder;

c++/src/kj/exception.c++

@@ -195,6 +195,7 @@ void printStackTraceOnCrash() {
   KJ_SYSCALL(sigaction(SIGBUS, &action, nullptr));
   KJ_SYSCALL(sigaction(SIGFPE, &action, nullptr));
   KJ_SYSCALL(sigaction(SIGABRT, &action, nullptr));
+  KJ_SYSCALL(sigaction(SIGILL, &action, nullptr));
 
   // Dump stack on unimplemented syscalls -- useful in seccomp sandboxes.
   KJ_SYSCALL(sigaction(SIGSYS, &action, nullptr));