Improve heap allocation.

c++/src/capnproto/compiler/capnpc-capnp.c++

@@ -95,7 +95,7 @@ TextBlob::TextBlob(Params&&... params) {
 }
 
 TextBlob::TextBlob(kj::Array<TextBlob>&& params) {
-  branches = kj::newArray<Branch>(params.size());
+  branches = kj::heapArray<Branch>(params.size());
   for (size_t i = 0; i < params.size(); i++) {
     branches[i].pos = nullptr;
     branches[i].content = kj::mv(params[i]);
@@ -113,8 +113,8 @@ void TextBlob::writeTo(kj::OutputStream& out) const {
 }
 
 void TextBlob::allocate(size_t textSize, size_t branchCount) {
-  text = kj::newArray<char>(textSize);
-  branches = kj::newArray<Branch>(branchCount);
+  text = kj::heapArray<char>(textSize);
+  branches = kj::heapArray<Branch>(branchCount);
 }
 
 template <typename First, typename... Rest>
@@ -160,7 +160,7 @@ TextBlob text(Params&&... params) {
 
 template <typename List, typename Func>
 TextBlob forText(List&& list, Func&& func) {
-  kj::Array<TextBlob> items = kj::newArray<TextBlob>(list.size());
+  kj::Array<TextBlob> items = kj::heapArray<TextBlob>(list.size());
   for (size_t i = 0; i < list.size(); i++) {
     items[i] = func(list[i]);
   }

c++/src/capnproto/schema-loader.c++

@@ -1096,7 +1096,7 @@ internal::RawSchema* SchemaLoader::Impl::tryGet(uint64_t typeId) const {
 }
 
 kj::Array<Schema> SchemaLoader::Impl::getAllLoaded() const {
-  kj::Array<Schema> result = kj::newArray<Schema>(schemas.size());
+  kj::Array<Schema> result = kj::heapArray<Schema>(schemas.size());
   size_t i = 0;
   for (auto& schema: schemas) {
     result[i++] = Schema(schema.second);

c++/src/capnproto/schema-loader.h

@@ -25,6 +25,7 @@
 #define CAPNPROTO_SCHEMA_LOADER_H_
 
 #include "schema.h"
+#include <kj/memory.h>
 
 namespace capnproto {
 

c++/src/capnproto/serialize-snappy.c++

@@ -61,7 +61,7 @@ private:
 SnappyInputStream::SnappyInputStream(BufferedInputStream& inner, kj::ArrayPtr<byte> buffer)
     : inner(inner) {
   if (buffer.size() < SNAPPY_BUFFER_SIZE) {
-    ownedBuffer = kj::newArray<byte>(SNAPPY_BUFFER_SIZE);
+    ownedBuffer = kj::heapArray<byte>(SNAPPY_BUFFER_SIZE);
     buffer = ownedBuffer;
   }
   this->buffer = buffer;
@@ -125,14 +125,14 @@ SnappyOutputStream::SnappyOutputStream(
       "snappy::MaxCompressedLength() changed?");
 
   if (buffer.size() < SNAPPY_BUFFER_SIZE) {
-    ownedBuffer = kj::newArray<byte>(SNAPPY_BUFFER_SIZE);
+    ownedBuffer = kj::heapArray<byte>(SNAPPY_BUFFER_SIZE);
     buffer = ownedBuffer;
   }
   this->buffer = buffer;
   bufferPos = buffer.begin();
 
   if (compressedBuffer.size() < SNAPPY_COMPRESSED_BUFFER_SIZE) {
-    ownedCompressedBuffer = kj::newArray<byte>(SNAPPY_COMPRESSED_BUFFER_SIZE);
+    ownedCompressedBuffer = kj::heapArray<byte>(SNAPPY_COMPRESSED_BUFFER_SIZE);
     compressedBuffer = ownedCompressedBuffer;
   }
   this->compressedBuffer = compressedBuffer;

c++/src/capnproto/serialize-test.c++

@@ -290,7 +290,7 @@ TEST(Serialize, FileDescriptors) {
 }
 
 TEST(Serialize, RejectTooManySegments) {
-  kj::Array<word> data = kj::newArray<word>(8192);
+  kj::Array<word> data = kj::heapArray<word>(8192);
   WireValue<uint32_t>* table = reinterpret_cast<WireValue<uint32_t>*>(data.begin());
   table[0].set(1024);
   for (uint i = 0; i < 1024; i++) {

c++/src/capnproto/serialize.c++

@@ -61,7 +61,7 @@ FlatArrayMessageReader::FlatArrayMessageReader(
   offset += segmentSize;
 
   if (segmentCount > 1) {
-    moreSegments = kj::newArray<kj::ArrayPtr<const word>>(segmentCount - 1);
+    moreSegments = kj::heapArray<kj::ArrayPtr<const word>>(segmentCount - 1);
 
     for (uint i = 1; i < segmentCount; i++) {
       uint segmentSize = table[i + 1].get();
@@ -96,7 +96,7 @@ kj::Array<word> messageToFlatArray(kj::ArrayPtr<const kj::ArrayPtr<const word>>
     totalSize += segment.size();
   }
 
-  kj::Array<word> result = kj::newArray<word>(totalSize);
+  kj::Array<word> result = kj::heapArray<word>(totalSize);
 
   internal::WireValue<uint32_t>* table =
       reinterpret_cast<internal::WireValue<uint32_t>*>(result.begin());
@@ -170,14 +170,14 @@ InputStreamMessageReader::InputStreamMessageReader(
   if (scratchSpace.size() < totalWords) {
     // TODO(perf):  Consider allocating each segment as a separate chunk to reduce memory
     //   fragmentation.
-    ownedSpace = kj::newArray<word>(totalWords);
+    ownedSpace = kj::heapArray<word>(totalWords);
     scratchSpace = ownedSpace;
   }
 
   segment0 = scratchSpace.slice(0, segment0Size);
 
   if (segmentCount > 1) {
-    moreSegments = kj::newArray<kj::ArrayPtr<const word>>(segmentCount - 1);
+    moreSegments = kj::heapArray<kj::ArrayPtr<const word>>(segmentCount - 1);
     size_t offset = segment0Size;
 
     for (uint i = 0; i < segmentCount - 1; i++) {

c++/src/kj/array-test.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "array.h"
+#include <gtest/gtest.h>
+#include "logging.h"
+#include <string>
+#include <list>
+
+namespace kj {
+namespace {
+
+struct TestObject {
+  TestObject() {
+    index = count;
+    CHECK(index != throwAt);
+    ++count;
+  }
+  TestObject(const TestObject& other) {
+    CHECK(other.index != throwAt);
+    index = -1;
+    copiedCount++;
+  }
+  ~TestObject() noexcept(false) {
+    if (index == -1) {
+      --copiedCount;
+    } else {
+      --count;
+      EXPECT_EQ(index, count);
+      CHECK(count != throwAt);
+    }
+  }
+
+  int index;
+
+  static int count;
+  static int copiedCount;
+  static int throwAt;
+};
+
+int TestObject::count = 0;
+int TestObject::copiedCount = 0;
+int TestObject::throwAt = -1;
+
+struct TestNoexceptObject {
+  TestNoexceptObject() noexcept {
+    index = count;
+    ++count;
+  }
+  TestNoexceptObject(const TestNoexceptObject& other) noexcept {
+    index = -1;
+    copiedCount++;
+  }
+  ~TestNoexceptObject() noexcept {
+    if (index == -1) {
+      --copiedCount;
+    } else {
+      --count;
+      EXPECT_EQ(index, count);
+    }
+  }
+
+  int index;
+
+  static int count;
+  static int copiedCount;
+};
+
+int TestNoexceptObject::count = 0;
+int TestNoexceptObject::copiedCount = 0;
+
+TEST(Array, TrivialConstructor) {
+  char* ptr;
+  {
+    Array<char> chars = heapArray<char>(32);
+    ptr = chars.begin();
+    chars[0] = 12;
+    chars[1] = 34;
+  }
+
+  {
+    Array<char> chars = heapArray<char>(32);
+
+    // Somewhat hacky:  We can't guarantee that the new array is allocated in the same place, but
+    // any reasonable allocator is highly likely to do so.  If it does, then we expect that the
+    // memory has not been initialized.
+    if (chars.begin() == ptr) {
+      EXPECT_NE(chars[0], 0);
+      EXPECT_NE(chars[1], 0);
+    }
+  }
+}
+
+TEST(Array, ComplexConstructor) {
+  TestObject::count = 0;
+  TestObject::throwAt = -1;
+
+  {
+    Array<TestObject> array = heapArray<TestObject>(32);
+    EXPECT_EQ(32, TestObject::count);
+  }
+  EXPECT_EQ(0, TestObject::count);
+}
+
+TEST(Array, ThrowingConstructor) {
+  TestObject::count = 0;
+  TestObject::throwAt = 16;
+
+  // If a constructor throws, the previous elements should still be destroyed.
+  EXPECT_ANY_THROW(heapArray<TestObject>(32));
+  EXPECT_EQ(0, TestObject::count);
+}
+
+TEST(Array, ThrowingDestructor) {
+  TestObject::count = 0;
+  TestObject::throwAt = -1;
+
+  Array<TestObject> array = heapArray<TestObject>(32);
+  EXPECT_EQ(32, TestObject::count);
+
+  // If a destructor throws, all elements should still be destroyed.
+  TestObject::throwAt = 16;
+  EXPECT_ANY_THROW(array = nullptr);
+  EXPECT_EQ(0, TestObject::count);
+}
+
+TEST(Array, AraryBuilder) {
+  TestObject::count = 0;
+  TestObject::throwAt = -1;
+
+  Array<TestObject> array;
+
+  {
+    ArrayBuilder<TestObject> builder = heapArrayBuilder<TestObject>(32);
+
+    for (uint i = 0; i < 32; i++) {
+      EXPECT_EQ(i, TestObject::count);
+      builder.add();
+    }
+
+    EXPECT_EQ(32, TestObject::count);
+    array = builder.finish();
+    EXPECT_EQ(32, TestObject::count);
+  }
+
+  EXPECT_EQ(32, TestObject::count);
+  array = nullptr;
+  EXPECT_EQ(0, TestObject::count);
+}
+
+TEST(Array, AraryBuilderAddAll) {
+  {
+    // Trivial case.
+    char text[] = "foo";
+    ArrayBuilder<char> builder = heapArrayBuilder<char>(5);
+    builder.add('<');
+    builder.addAll(text, text + 3);
+    builder.add('>');
+    auto array = builder.finish();
+    EXPECT_EQ("<foo>", std::string(array.begin(), array.end()));
+  }
+
+  {
+    // Trivial case, const.
+    const char* text = "foo";
+    ArrayBuilder<char> builder = heapArrayBuilder<char>(5);
+    builder.add('<');
+    builder.addAll(text, text + 3);
+    builder.add('>');
+    auto array = builder.finish();
+    EXPECT_EQ("<foo>", std::string(array.begin(), array.end()));
+  }
+
+  {
+    // Trivial case, non-pointer iterator.
+    std::list<char> text = {'f', 'o', 'o'};
+    ArrayBuilder<char> builder = heapArrayBuilder<char>(5);
+    builder.add('<');
+    builder.addAll(text);
+    builder.add('>');
+    auto array = builder.finish();
+    EXPECT_EQ("<foo>", std::string(array.begin(), array.end()));
+  }
+
+  {
+    // Complex case.
+    std::string strs[] = {"foo", "bar", "baz"};
+    ArrayBuilder<std::string> builder = heapArrayBuilder<std::string>(5);
+    builder.add("qux");
+    builder.addAll(strs, strs + 3);
+    builder.add("quux");
+    auto array = builder.finish();
+    EXPECT_EQ("qux", array[0]);
+    EXPECT_EQ("foo", array[1]);
+    EXPECT_EQ("bar", array[2]);
+    EXPECT_EQ("baz", array[3]);
+    EXPECT_EQ("quux", array[4]);
+  }
+
+  {
+    // Complex case, noexcept.
+    TestNoexceptObject::count = 0;
+    TestNoexceptObject::copiedCount = 0;
+    TestNoexceptObject objs[3];
+    EXPECT_EQ(3, TestNoexceptObject::count);
+    EXPECT_EQ(0, TestNoexceptObject::copiedCount);
+    ArrayBuilder<TestNoexceptObject> builder = heapArrayBuilder<TestNoexceptObject>(3);
+    EXPECT_EQ(3, TestNoexceptObject::count);
+    EXPECT_EQ(0, TestNoexceptObject::copiedCount);
+    builder.addAll(objs, objs + 3);
+    EXPECT_EQ(3, TestNoexceptObject::count);
+    EXPECT_EQ(3, TestNoexceptObject::copiedCount);
+    auto array = builder.finish();
+    EXPECT_EQ(3, TestNoexceptObject::count);
+    EXPECT_EQ(3, TestNoexceptObject::copiedCount);
+  }
+  EXPECT_EQ(0, TestNoexceptObject::count);
+  EXPECT_EQ(0, TestNoexceptObject::copiedCount);
+
+  {
+    // Complex case, exceptions possible.
+    TestObject::count = 0;
+    TestObject::copiedCount = 0;
+    TestObject::throwAt = -1;
+    TestObject objs[3];
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(0, TestObject::copiedCount);
+    ArrayBuilder<TestObject> builder = heapArrayBuilder<TestObject>(3);
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(0, TestObject::copiedCount);
+    builder.addAll(objs, objs + 3);
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(3, TestObject::copiedCount);
+    auto array = builder.finish();
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(3, TestObject::copiedCount);
+  }
+  EXPECT_EQ(0, TestObject::count);
+  EXPECT_EQ(0, TestObject::copiedCount);
+
+  {
+    // Complex case, exceptions occur.
+    TestObject::count = 0;
+    TestObject::copiedCount = 0;
+    TestObject::throwAt = -1;
+    TestObject objs[3];
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(0, TestObject::copiedCount);
+
+    TestObject::throwAt = 1;
+
+    ArrayBuilder<TestObject> builder = heapArrayBuilder<TestObject>(3);
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(0, TestObject::copiedCount);
+
+    EXPECT_ANY_THROW(builder.addAll(objs, objs + 3));
+    TestObject::throwAt = -1;
+
+    EXPECT_EQ(3, TestObject::count);
+    EXPECT_EQ(0, TestObject::copiedCount);
+  }
+  EXPECT_EQ(0, TestObject::count);
+  EXPECT_EQ(0, TestObject::copiedCount);
+}
+
+}  // namespace
+}  // namespace kj

c++/src/kj/array.c++

@@ -22,7 +22,87 @@
 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "array.h"
+#include <iostream>
 
 namespace kj {
 
+ArrayDisposer::~ArrayDisposer() {}
+
+namespace internal {
+
+struct HeapArrayDisposer::ExceptionGuard {
+  byte* pos;
+  size_t elementSize;
+  size_t elementCount;
+  size_t constructedCount;
+  void (*destroyElement)(void*);
+
+  ExceptionGuard(void* ptr, size_t elementSize, size_t elementCount,
+                 void (*destroyElement)(void*))
+      : pos(reinterpret_cast<byte*>(ptr) + elementSize * elementCount),
+        elementSize(elementSize), elementCount(elementCount),
+        destroyElement(destroyElement) {}
+
+  ~ExceptionGuard() {
+    if (pos != nullptr) {
+      destroyAll();
+      operator delete(pos);
+    }
+  }
+
+  void destroyAll() {
+    while (elementCount > 0) {
+      pos -= elementSize;
+      --elementCount;
+      destroyElement(pos);
+    }
+  }
+};
+
+void* HeapArrayDisposer::allocateImpl(size_t elementSize, size_t elementCount, size_t capacity,
+                                      void (*constructElement)(void*),
+                                      void (*destroyElement)(void*)) {
+  void* result = operator new(elementSize * capacity);
+
+  if (constructElement == nullptr) {
+    // Nothing to do.
+  } else if (destroyElement == nullptr) {
+    byte* pos = reinterpret_cast<byte*>(result);
+    while (elementCount > 0) {
+      constructElement(pos);
+      pos += elementSize;
+      --elementCount;
+    }
+  } else {
+    ExceptionGuard guard(result, elementSize, 0, destroyElement);
+    while (guard.elementCount < elementCount) {
+      constructElement(guard.pos);
+      guard.pos += elementSize;
+      ++guard.elementCount;
+    }
+    guard.pos = nullptr;
+  }
+
+  return result;
+}
+
+void HeapArrayDisposer::disposeImpl(
+    void* firstElement, size_t elementSize, size_t elementCount, size_t capacity,
+    void (*destroyElement)(void*)) const {
+  // Note that capacity is ignored since operator delete() doesn't care about it.
+
+  if (destroyElement == nullptr) {
+    operator delete(firstElement);
+  } else {
+    ExceptionGuard guard(firstElement, elementSize, elementCount, destroyElement);
+    guard.destroyAll();
+
+    // If an exception is thrown, we'll continue the destruction process in ExceptionGuard's
+    // destructor.  If _that_ throws an exception, the program terminates according to C++ rules.
+  }
+}
+
+const HeapArrayDisposer HeapArrayDisposer::instance = HeapArrayDisposer();
+
+}  // namespace internal
 }  // namespace kj

c++/src/kj/array.h

@@ -25,29 +25,62 @@
 #define KJ_ARRAY_H_
 
 #include "common.h"
-#include "memory.h"
 #include <string.h>
 
 namespace kj {
 
+// =======================================================================================
+// ArrayDisposer -- Implementation details.
+
+class ArrayDisposer {
+  // Much like Disposer from memory.h.
+
+protected:
+  virtual ~ArrayDisposer();
+
+  virtual void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
+                           size_t capacity, void (*destroyElement)(void*)) const = 0;
+  // Disposes of the array.  `destroyElement` invokes the destructor of each element, or is nullptr
+  // if the elements have trivial destructors.  `capacity` is the amount of space that was
+  // allocated while `elementCount` is the number of elements that were actually constructed;
+  // these are always the same number for Array<T> but may be different when using ArrayBuilder<T>.
+
+public:
+
+  template <typename T>
+  void dispose(T* firstElement, size_t elementCount, size_t capacity) const;
+  // Helper wrapper around disposeImpl().
+  //
+  // Callers must not call dispose() on the same array twice, even if the first call throws
+  // an exception.
+
+private:
+  template <typename T, bool hasTrivialDestructor = __has_trivial_destructor(T)>
+  struct Dispose_;
+};
+
 // =======================================================================================
 // Array
 
 template <typename T>
 class Array {
-  // An owned array which will automatically be deleted in the destructor.  Can be moved, but not
-  // copied.
+  // An owned array which will automatically be disposed of (using an ArrayDisposer) in the
+  // destructor.  Can be moved, but not copied.  Much like Own<T>, but for arrays rather than
+  // single objects.
 
 public:
   inline Array(): ptr(nullptr), size_(0) {}
   inline Array(decltype(nullptr)): ptr(nullptr), size_(0) {}
-  inline Array(Array&& other) noexcept: ptr(other.ptr), size_(other.size_) {
+  inline Array(Array&& other) noexcept
+      : ptr(other.ptr), size_(other.size_), disposer(other.disposer) {
     other.ptr = nullptr;
     other.size_ = 0;
   }
+  inline Array(T* firstElement, size_t size, const ArrayDisposer& disposer)
+      : ptr(firstElement), size_(size), disposer(&disposer) {}
 
   KJ_DISALLOW_COPY(Array);
-  inline ~Array() noexcept { delete[] ptr; }
+  inline ~Array() noexcept { dispose(); }
 
   inline operator ArrayPtr<T>() {
     return ArrayPtr<T>(ptr, size_);
@@ -65,10 +98,14 @@ public:
     return ptr[index];
   }
 
-  inline T* begin() const { return ptr; }
-  inline T* end() const { return ptr + size_; }
-  inline T& front() const { return *ptr; }
-  inline T& back() const { return *(ptr + size_ - 1); }
+  inline const T* begin() const { return ptr; }
+  inline const T* end() const { return ptr + size_; }
+  inline const T& front() const { return *ptr; }
+  inline const T& back() const { return *(ptr + size_ - 1); }
+  inline T* begin() { return ptr; }
+  inline T* end() { return ptr + size_; }
+  inline T& front() { return *ptr; }
+  inline T& back() { return *(ptr + size_ - 1); }
 
   inline ArrayPtr<T> slice(size_t start, size_t end) {
     KJ_INLINE_DPRECOND(start <= end && end <= size_, "Out-of-bounds Array::slice().");
@@ -83,16 +120,15 @@ public:
   inline bool operator!=(decltype(nullptr)) const { return size_ != 0; }
 
   inline Array& operator=(decltype(nullptr)) {
-    delete[] ptr;
-    ptr = nullptr;
-    size_ = 0;
+    dispose();
     return *this;
   }
 
   inline Array& operator=(Array&& other) {
-    delete[] ptr;
+    dispose();
     ptr = other.ptr;
     size_ = other.size_;
+    disposer = other.disposer;
     other.ptr = nullptr;
     other.size_ = 0;
     return *this;
@@ -101,20 +137,56 @@ public:
 private:
   T* ptr;
   size_t size_;
+  const ArrayDisposer* disposer;
+
+  inline void dispose() {
+    // Make sure that if an exception is thrown, we are left with a null ptr, so we won't possibly
+    // dispose again.
+    T* ptrCopy = ptr;
+    size_t sizeCopy = size_;
+    if (ptrCopy != nullptr) {
+      ptr = nullptr;
+      size_ = 0;
+      disposer->dispose(ptrCopy, sizeCopy, sizeCopy);
+    }
+  }
+};
+
+namespace internal {
+
+class HeapArrayDisposer final: public ArrayDisposer {
+public:
+  static void* allocateImpl(size_t elementSize, size_t elementCount, size_t capacity,
+                            void (*constructElement)(void*), void (*destroyElement)(void*));
+  // Allocates and constructs the array.  Both function pointers are null if the constructor is
+  // trivial, otherwise destroyElement is null if the constructor doesn't throw.
+
+  virtual void disposeImpl(void* firstElement, size_t elementSize, size_t elementCount,
+                           size_t capacity, void (*destroyElement)(void*)) const override;
 
-  inline explicit Array(size_t size): ptr(new T[size]), size_(size) {}
-  inline Array(T* ptr, size_t size): ptr(ptr), size_(size) {}
+  template <typename T>
+  static T* allocate(size_t count);
+  template <typename T>
+  static T* allocateUninitialized(size_t count);
 
-  template <typename U>
-  friend Array<U> newArray(size_t size);
+  static const HeapArrayDisposer instance;
+
+private:
+  template <typename T, bool hasTrivialConstructor = __has_trivial_constructor(T),
+                        bool hasNothrowConstructor = __has_nothrow_constructor(T)>
+  struct Allocate_;
 
-  template <typename U>
-  friend class ArrayBuilder;
+  struct ExceptionGuard;
 };
 
+}  // namespace internal
+
 template <typename T>
-inline Array<T> newArray(size_t size) {
-  return Array<T>(size);
+inline Array<T> heapArray(size_t size) {
+  // Much like `heap<T>()` from memory.h, allocates a new array on the heap.
+
+  return Array<T>(internal::HeapArrayDisposer::allocate<T>(size), size,
+                  internal::HeapArrayDisposer::instance);
 }
 
 // =======================================================================================
@@ -122,53 +194,90 @@ inline Array<T> newArray(size_t size) {
 
 template <typename T>
 class ArrayBuilder {
-  // TODO(cleanup):  This class doesn't work for non-primitive types because Slot is not
-  //   constructable.  Giving Slot a constructor/destructor means arrays of it have to be tagged
-  //   so operator delete can run the destructors.  If we reinterpret_cast the array to an array
-  //   of T and delete it as that type, operator delete gets very upset.
-  //
-  //   Perhaps we should bite the bullet and make the Array family do manual memory allocation,
-  //   bypassing the rather-stupid C++ array new/delete operators which store a redundant copy of
-  //   the size anyway.
-
-  union Slot {
-    T value;
-    char dummy;
-  };
-  static_assert(sizeof(Slot) == sizeof(T), "union is bigger than content?");
+  // Class which lets you build an Array<T> specifying the exact constructor arguments for each
+  // element, rather than starting by default-constructing them.
 
 public:
-  explicit ArrayBuilder(size_t size): ptr(new Slot[size]), pos(ptr), endPtr(ptr + size) {}
-  ~ArrayBuilder() {
-    for (Slot* p = ptr; p < pos; ++p) {
-      p->value.~T();
-    }
-    delete [] ptr;
+  ArrayBuilder(): ptr(nullptr), pos(nullptr), endPtr(nullptr) {}
+  ArrayBuilder(decltype(nullptr)): ptr(nullptr), pos(nullptr), endPtr(nullptr) {}
+  explicit ArrayBuilder(T* firstElement, size_t capacity, const ArrayDisposer& disposer)
+      : ptr(firstElement), pos(firstElement), endPtr(firstElement + capacity),
+        disposer(&disposer) {}
+  ArrayBuilder(ArrayBuilder&& other)
+      : ptr(other.ptr), pos(other.pos), endPtr(other.endPtr), disposer(other.disposer) {
+    other.ptr = nullptr;
+    other.pos = nullptr;
+    other.endPtr = nullptr;
+  }
+  KJ_DISALLOW_COPY(ArrayBuilder);
+  inline ~ArrayBuilder() { dispose(); }
+
+  inline operator ArrayPtr<T>() {
+    return arrayPtr(ptr, pos);
+  }
+  inline operator ArrayPtr<const T>() const {
+    return arrayPtr(ptr, pos);
+  }
+  inline ArrayPtr<T> asPtr() {
+    return arrayPtr(ptr, pos);
+  }
+
+  inline size_t size() const { return pos - ptr; }
+  inline size_t capacity() const { return endPtr - ptr; }
+  inline T& operator[](size_t index) const {
+    KJ_INLINE_DPRECOND(index < pos - ptr, "Out-of-bounds Array access.");
+    return ptr[index];
+  }
+
+  inline const T* begin() const { return ptr; }
+  inline const T* end() const { return pos; }
+  inline const T& front() const { return *ptr; }
+  inline const T& back() const { return *(pos - 1); }
+  inline T* begin() { return ptr; }
+  inline T* end() { return pos; }
+  inline T& front() { return *ptr; }
+  inline T& back() { return *(pos - 1); }
+
+  ArrayBuilder& operator=(ArrayBuilder&& other) {
+    dispose();
+    ptr = other.ptr;
+    pos = other.pos;
+    endPtr = other.endPtr;
+    disposer = other.disposer;
+    other.ptr = nullptr;
+    other.pos = nullptr;
+    other.endPtr = nullptr;
+    return *this;
+  }
+  ArrayBuilder& operator=(decltype(nullptr)) {
+    dispose();
+    return *this;
   }
 
   template <typename... Params>
   void add(Params&&... params) {
     KJ_INLINE_DPRECOND(pos < endPtr, "Added too many elements to ArrayBuilder.");
-    new(&pos->value) T(kj::fwd<Params>(params)...);
+    ctor(*pos, kj::fwd<Params>(params)...);
     ++pos;
   }
 
   template <typename Container>
   void addAll(Container&& container) {
-    Slot* __restrict__ pos_ = pos;
-    auto i = container.begin();
-    auto end = container.end();
-    while (i != end) {
-      pos_++->value = *i++;
-    }
-    pos = pos_;
+    addAll(container.begin(), container.end());
   }
 
+  template <typename Iterator>
+  void addAll(Iterator start, Iterator end);
+
   Array<T> finish() {
-    // We could allow partial builds if Array<T> used a deleter callback, but that would make
-    // Array<T> bigger for no benefit most of the time.
+    // We could safely remove this check as long as HeapArrayDisposer relies on operator delete
+    // (which doesn't need to know the original capacity) or if we created a custom disposer for
+    // ArrayBuilder which stores the capacity in a prefix.  But that would mean we can't allow
+    // arbitrary disposers with ArrayBuilder in the future, and anyway this check might catch bugs.
+    // Probably we should just create a new Vector-like data structure if we want to allow building
+    // of arrays without knowing the final size in advance.
     KJ_INLINE_DPRECOND(pos == endPtr, "ArrayBuilder::finish() called prematurely.");
-    Array<T> result(reinterpret_cast<T*>(ptr), pos - ptr);
+    Array<T> result(reinterpret_cast<T*>(ptr), pos - ptr, internal::HeapArrayDisposer::instance);
     ptr = nullptr;
     pos = nullptr;
     endPtr = nullptr;
@@ -176,11 +285,183 @@ public:
   }
 
 private:
-  Slot* ptr;
-  Slot* pos;
-  Slot* endPtr;
+  T* ptr;
+  T* pos;
+  T* endPtr;
+  const ArrayDisposer* disposer;
+
+  inline void dispose() {
+    // Make sure that if an exception is thrown, we are left with a null ptr, so we won't possibly
+    // dispose again.
+    T* ptrCopy = ptr;
+    T* posCopy = pos;
+    T* endCopy = endPtr;
+    if (ptrCopy != nullptr) {
+      ptr = nullptr;
+      pos = nullptr;
+      endPtr = nullptr;
+      disposer->dispose(ptrCopy, posCopy - ptrCopy, endCopy - ptrCopy);
+    }
+  }
 };
 
+template <typename T>
+inline ArrayBuilder<T> heapArrayBuilder(size_t size) {
+  // Like `heapArray<T>()` but does not default-construct the elements.  You must construct them
+  // manually by calling `add()`.
+
+  return ArrayBuilder<T>(internal::HeapArrayDisposer::allocateUninitialized<T>(size), size,
+                         internal::HeapArrayDisposer::instance);
+}
+
+// =======================================================================================
+// Inline implementation details
+
+template <typename T>
+struct ArrayDisposer::Dispose_<T, true> {
+  static void dispose(T* firstElement, size_t elementCount, size_t capacity,
+                      const ArrayDisposer& disposer) {
+    disposer.disposeImpl(firstElement, sizeof(T), elementCount, capacity, nullptr);
+  }
+};
+template <typename T>
+struct ArrayDisposer::Dispose_<T, false> {
+  static void destruct(void* ptr) {
+    kj::dtor(*reinterpret_cast<T*>(ptr));
+  }
+
+  static void dispose(T* firstElement, size_t elementCount, size_t capacity,
+                      const ArrayDisposer& disposer) {
+    disposer.disposeImpl(firstElement, sizeof(T), elementCount, capacity, &destruct);
+  }
+};
+
+template <typename T>
+void ArrayDisposer::dispose(T* firstElement, size_t elementCount, size_t capacity) const {
+  Dispose_<T>::dispose(firstElement, elementCount, capacity, *this);
+}
+
+namespace internal {
+
+template <typename T>
+struct HeapArrayDisposer::Allocate_<T, true, true> {
+  static T* allocate(size_t elementCount, size_t capacity) {
+    return reinterpret_cast<T*>(allocateImpl(
+        sizeof(T), elementCount, capacity, nullptr, nullptr));
+  }
+};
+template <typename T>
+struct HeapArrayDisposer::Allocate_<T, false, true> {
+  static void construct(void* ptr) {
+    kj::ctor(*reinterpret_cast<T*>(ptr));
+  }
+  static T* allocate(size_t elementCount, size_t capacity) {
+    return reinterpret_cast<T*>(allocateImpl(
+        sizeof(T), elementCount, capacity, &construct, nullptr));
+  }
+};
+template <typename T>
+struct HeapArrayDisposer::Allocate_<T, false, false> {
+  static void construct(void* ptr) {
+    kj::ctor(*reinterpret_cast<T*>(ptr));
+  }
+  static void destruct(void* ptr) {
+    kj::dtor(*reinterpret_cast<T*>(ptr));
+  }
+  static T* allocate(size_t elementCount, size_t capacity) {
+    return reinterpret_cast<T*>(allocateImpl(
+        sizeof(T), elementCount, capacity, &construct, &destruct));
+  }
+};
+
+template <typename T>
+T* HeapArrayDisposer::allocate(size_t count) {
+  return Allocate_<T>::allocate(count, count);
+}
+
+template <typename T>
+T* HeapArrayDisposer::allocateUninitialized(size_t count) {
+  return Allocate_<T, true, true>::allocate(0, count);
+}
+
+template <typename Element, typename Iterator,
+          bool trivial = __has_trivial_copy(Element) && __has_trivial_assign(Element)>
+struct CopyConstructArray_;
+
+template <typename T>
+struct CopyConstructArray_<T, T*, true> {
+  static inline T* apply(T* __restrict__ pos, T* start, T* end) {
+    memcpy(pos, start, end - start);
+    return pos + (end - start);
+  }
+};
+
+template <typename T>
+struct CopyConstructArray_<T, const T*, true> {
+  static inline T* apply(T* __restrict__ pos, const T* start, const T* end) {
+    memcpy(pos, start, end - start);
+    return pos + (end - start);
+  }
+};
+
+template <typename T, typename Iterator>
+struct CopyConstructArray_<T, Iterator, true> {
+  static inline T* apply(T* __restrict__ pos, Iterator start, Iterator end) {
+    // Since both the copy constructor and assignment operator are trivial, we know that assignment
+    // is equivalent to copy-constructing.  So we can make this case somewhat easier for the
+    // compiler to optimize.
+    while (start != end) {
+      *pos++ = *start++;
+    }
+    return pos;
+  }
+};
+
+template <typename T, typename Iterator>
+struct CopyConstructArray_<T, Iterator, false> {
+  struct ExceptionGuard {
+    T* start;
+    T* pos;
+    inline explicit ExceptionGuard(T* pos): start(pos), pos(pos) {}
+    ~ExceptionGuard() {
+      while (pos > start) {
+        dtor(*--pos);
+      }
+    }
+  };
+
+  static T* apply(T* __restrict__ pos, Iterator start, Iterator end) {
+    if (noexcept(T(instance<const T&>()))) {
+      while (start != end) {
+        ctor(*pos++, upcast<const T&>(*start++));
+      }
+      return pos;
+    } else {
+      // Crap.  This is complicated.
+      ExceptionGuard guard(pos);
+      while (start != end) {
+        ctor(*guard.pos, upcast<const T&>(*start++));
+        ++guard.pos;
+      }
+      guard.start = guard.pos;
+      return guard.pos;
+    }
+  }
+};
+
+template <typename T, typename Iterator>
+inline T* copyConstructArray(T* dst, Iterator start, Iterator end) {
+  return CopyConstructArray_<T, RemoveReference<Iterator>>::apply(dst, start, end);
+}
+
+}  // namespace internal
+
+template <typename T>
+template <typename Iterator>
+void ArrayBuilder<T>::addAll(Iterator start, Iterator end) {
+  pos = internal::copyConstructArray(pos, start, end);
+}
+
 }  // namespace kj
 
 #endif  // KJ_ARRAY_H_

c++/src/kj/common.h

@@ -145,7 +145,7 @@ void inlinePreconditionFailure(
   bool name##_isOnStack = name##_size <= (minStack); \
   type name##_stack[minStack]; \
   ::kj::Array<type> name##_heap = name##_isOnStack ? \
-      nullptr : kj::newArray<type>(name##_size); \
+      nullptr : kj::heapArray<type>(name##_size); \
   ::kj::ArrayPtr<type> name = name##_isOnStack ? \
       kj::arrayPtr(name##_stack, name##_size) : name##_heap
 #else
@@ -154,7 +154,7 @@ void inlinePreconditionFailure(
   bool name##_isOnStack = name##_size <= (maxStack); \
   type name##_stack[name##_isOnStack ? size : 0]; \
   ::kj::Array<type> name##_heap = name##_isOnStack ? \
-      nullptr : kj::newArray<type>(name##_size); \
+      nullptr : kj::heapArray<type>(name##_size); \
   ::kj::ArrayPtr<type> name = name##_isOnStack ? \
       kj::arrayPtr(name##_stack, name##_size) : name##_heap
 #endif

c++/src/kj/exception.c++

@@ -98,7 +98,7 @@ const char* Exception::what() const noexcept {
     }
   }
 
-  Array<Array<char>> contextText = newArray<Array<char>>(contextDepth);
+  Array<Array<char>> contextText = heapArray<Array<char>>(contextDepth);
 
   contextDepth = 0;
   contextPtr = &context;

c++/src/kj/exception.h

@@ -25,6 +25,7 @@
 #define KJ_EXCEPTION_H_
 
 #include <exception>
+#include "memory.h"
 #include "array.h"
 
 namespace kj {

c++/src/kj/io.c++

@@ -52,7 +52,7 @@ void OutputStream::write(ArrayPtr<const ArrayPtr<const byte>> pieces) {
 // =======================================================================================
 
 BufferedInputStreamWrapper::BufferedInputStreamWrapper(InputStream& inner, ArrayPtr<byte> buffer)
-    : inner(inner), ownedBuffer(buffer == nullptr ? newArray<byte>(8192) : nullptr),
+    : inner(inner), ownedBuffer(buffer == nullptr ? heapArray<byte>(8192) : nullptr),
       buffer(buffer == nullptr ? ownedBuffer : buffer) {}
 
 BufferedInputStreamWrapper::~BufferedInputStreamWrapper() {}
@@ -118,7 +118,7 @@ void BufferedInputStreamWrapper::skip(size_t bytes) {
 
 BufferedOutputStreamWrapper::BufferedOutputStreamWrapper(OutputStream& inner, ArrayPtr<byte> buffer)
     : inner(inner),
-      ownedBuffer(buffer == nullptr ? newArray<byte>(8192) : nullptr),
+      ownedBuffer(buffer == nullptr ? heapArray<byte>(8192) : nullptr),
       buffer(buffer == nullptr ? ownedBuffer : buffer),
       bufferPos(this->buffer.begin()) {}
 

c++/src/kj/logging.c++

@@ -157,7 +157,7 @@ static Array<char> makeDescription(DescriptionStyle style, const char* code, int
       totalSize += argValues[i].size();
     }
 
-    ArrayBuilder<char> result(totalSize);
+    ArrayBuilder<char> result = heapArrayBuilder<char>(totalSize);
 
     switch (style) {
       case LOG:

c++/src/kj/memory.h

@@ -29,27 +29,40 @@
 namespace kj {
 
 // =======================================================================================
+// Disposer -- Implementation details.
 
 class Disposer {
-  // Abstract interface for a thing that disposes of some other object.  Often, it makes sense to
-  // decouple an object from the knowledge of how to dispose of it.
+  // Abstract interface for a thing that "disposes" of objects, where "disposing" usually means
+  // calling the destructor followed by freeing the underlying memory.  `Own<T>` encapsulates an
+  // object pointer with corresponding Disposer.
+  //
+  // Few developers will ever touch this interface.  It is primarily useful for those implementing
+  // custom memory allocators.
 
 protected:
   virtual ~Disposer();
 
+  virtual void disposeImpl(void* pointer) const = 0;
+  // Disposes of the object, given a pointer to the beginning of the object.  If the object is
+  // polymorphic, this pointer is determined by dynamic_cast<void*>().  For non-polymorphic types,
+  // Own<T> does not allow any casting, so the pointer exactly matches the original one given to
+  // Own<T>.
+
 public:
-  virtual void dispose(void* interiorPointer) = 0;
-  // Disposes of the object that this Disposer owns, and possibly disposes of the disposer itself.
+
+  template <typename T>
+  void dispose(T* object) const;
+  // Helper wrapper around disposeImpl().
   //
-  // Callers must assume that the Disposer itself is no longer valid once this returns -- e.g. it
-  // might delete itself.  Callers must in particular be sure not to call the Disposer again even
-  // when dispose() throws an exception.
+  // If T is polymorphic, calls `disposeImpl(dynamic_cast<void*>(object))`, otherwise calls
+  // `disposeImpl(upcast<void*>(object))`.
   //
-  // `interiorPointer` points somewhere inside of the object -- NOT necessarily at the beginning,
-  // especially in the presence of multiple inheritance.  Most implementations should ignore the
-  // pointer, though a tricky memory allocator could get away with sharing one Disposer among
-  // multiple objects if it can figure out how to find the beginning of the object given an
-  // arbitrary interior pointer.
+  // Callers must not call dispose() on the same pointer twice, even if the first call throws
+  // an exception.
+
+private:
+  template <typename T, bool polymorphic = __is_polymorphic(T)>
+  struct Dispose_;
 };
 
 // =======================================================================================
@@ -64,11 +77,12 @@ class Own {
   // This is much like std::unique_ptr, except:
   // - You cannot release().  An owned object is not necessarily allocated with new (see next
   //   point), so it would be hard to use release() correctly.
-  // - The deleter is made polymorphic by virtual call rather than by template.  This is a much
-  //   more powerful default -- it allows any random module to decide to use a custom allocator.
-  //   This could be accomplished with unique_ptr by forcing everyone to use e.g.
-  //   std::unique_ptr<T, kj::Disposer&>, but at that point we've lost basically any benefit
-  //   of interoperating with std::unique_ptr anyway.
+  // - The deleter is made polymorphic by virtual call rather than by template.  This is much
+  //   more powerful -- it allows the use of custom allocators, freelists, etc.  This could
+  //   _almost_ be accomplished with unique_ptr by forcing everyone to use something like
+  //   std::unique_ptr<T, kj::Deleter>, except that things get hairy in the presence of multiple
+  //   inheritance and upcasting, and anyway if you force everyone to use a custom deleter
+  //   then you've lost any benefit to interoperating with the "standard" unique_ptr.
 
 public:
   Own(const Own& other) = delete;
@@ -76,8 +90,12 @@ public:
       : disposer(other.disposer), ptr(other.ptr) { other.ptr = nullptr; }
   template <typename U>
   inline Own(Own<U>&& other) noexcept
-      : disposer(other.disposer), ptr(other.ptr) { other.ptr = nullptr; }
-  inline Own(T* ptr, Disposer* disposer) noexcept: disposer(disposer), ptr(ptr) {}
+      : disposer(other.disposer), ptr(other.ptr) {
+    static_assert(__is_polymorphic(T),
+        "Casting owned pointers requires that the target type is polymorphic.");
+    other.ptr = nullptr;
+  }
+  inline Own(T* ptr, const Disposer& disposer) noexcept: disposer(&disposer), ptr(ptr) {}
 
   ~Own() noexcept { dispose(); }
 
@@ -99,13 +117,13 @@ public:
   inline operator const T*() const { return ptr; }
 
 private:
-  Disposer* disposer;  // Only valid if ptr != nullptr.
+  const Disposer* disposer;  // Only valid if ptr != nullptr.
   T* ptr;
 
   inline void dispose() {
     // Make sure that if an exception is thrown, we are left with a null ptr, so we won't possibly
     // dispose again.
-    void* ptrCopy = ptr;
+    T* ptrCopy = ptr;
     if (ptrCopy != nullptr) {
       ptr = nullptr;
       disposer->dispose(ptrCopy);
@@ -116,26 +134,50 @@ private:
 namespace internal {
 
 template <typename T>
-class HeapValue final: public Disposer {
+class HeapDisposer final: public Disposer {
 public:
-  template <typename... Params>
-  inline HeapValue(Params&&... params): value(kj::fwd<Params>(params)...) {}
-
-  virtual void dispose(void*) override { delete this; }
+  virtual void disposeImpl(void* pointer) const override { delete reinterpret_cast<T*>(pointer); }
 
-  T value;
+  static const HeapDisposer instance;
 };
 
+template <typename T>
+const HeapDisposer<T> HeapDisposer<T>::instance = HeapDisposer<T>();
+
 }  // namespace internal
 
 template <typename T, typename... Params>
 Own<T> heap(Params&&... params) {
   // heap<T>(...) allocates a T on the heap, forwarding the parameters to its constructor.  The
   // exact heap implementation is unspecified -- for now it is operator new, but you should not
-  // assume anything.
+  // assume this.  (Since we know the object size at delete time, we could actually implement an
+  // allocator that is more efficient than operator new.)
 
-  auto result = new internal::HeapValue<T>(kj::fwd<Params>(params)...);
-  return Own<T>(&result->value, result);
+  return Own<T>(new T(kj::fwd<Params>(params)...), internal::HeapDisposer<T>::instance);
+}
+
+// =======================================================================================
+// Inline implementation details
+
+template <typename T>
+struct Disposer::Dispose_<T, true> {
+  static void dispose(T* object, const Disposer& disposer) {
+    // Note that dynamic_cast<void*> does not require RTTI to be enabled, because the offset to
+    // the top of the object is in the vtable -- as it obviously needs to be to correctly implement
+    // operator delete.
+    disposer.disposeImpl(dynamic_cast<void*>(object));
+  }
+};
+template <typename T>
+struct Disposer::Dispose_<T, false> {
+  static void dispose(T* object, const Disposer& disposer) {
+    disposer.disposeImpl(static_cast<void*>(object));
+  }
+};
+
+template <typename T>
+void Disposer::dispose(T* object) const {
+  Dispose_<T>::dispose(object, *this);
 }
 
 }  // namespace kj

c++/src/kj/string.c++

@@ -25,11 +25,11 @@
 
 namespace kj {
 
-String::String(const char* value): content(newArray<char>(strlen(value) + 1)) {
+String::String(const char* value): content(heapArray<char>(strlen(value) + 1)) {
   strcpy(content.begin(), value);
 }
 
-String::String(const char* value, size_t length): content(newArray<char>(length + 1)) {
+String::String(const char* value, size_t length): content(heapArray<char>(length + 1)) {
   memcpy(content.begin(), value, length);
   content[length] = '\0';
 }

c++/src/kj/util.h

@@ -98,7 +98,7 @@ template <typename T, typename Container>
 Array<T> iterableToArray(Container&& a) {
   // Converts an arbitrary iterable container into an array of the given element type.
 
-  Array<T> result = newArray<T>(a.size());
+  Array<T> result = heapArray<T>(a.size());
   auto i = a.iterator();
   auto end = a.end();
   T* __restrict__ ptr = result.begin();
@@ -146,7 +146,7 @@ Array<Element> concat(Params&&... params) {
   // Eclipse reports a bogus error on `size()`.
   Array<Element> result;
 #else
-  Array<Element> result = newArray<Element>(sum({params.size()...}));
+  Array<Element> result = heapArray<Element>(sum({params.size()...}));
 #endif
   fill(result.begin(), std::forward<Params>(params)...);
   return result;
@@ -230,7 +230,7 @@ Array<char> strArray(T&& arr, const char* delim) {
     size += pieces[i].size();
   }
 
-  Array<char> result = newArray<char>(size);
+  Array<char> result = heapArray<char>(size);
   char* pos = result.begin();
   for (size_t i = 0; i < arr.size(); i++) {
     if (i > 0) {
@@ -255,7 +255,7 @@ inline Array<char> Stringifier::operator*(const Array<T>& arr) const {
 template <typename T, typename Func>
 auto mapArray(T&& arr, Func&& func) -> Array<decltype(func(arr[0]))> {
   // TODO(cleanup):  Use ArrayBuilder.
-  Array<decltype(func(arr[0]))> result = newArray<decltype(func(arr[0]))>(arr.size());
+  Array<decltype(func(arr[0]))> result = heapArray<decltype(func(arr[0]))>(arr.size());
   size_t pos = 0;
   for (auto& element: arr) {
     result[pos++] = func(element);