Implement kj::Table, an alternative to STL maps/sets.

Hash-based (unordered) and tree-based (ordered) indexing are provided.

kj::Table offers advantages over STL:
- A Table can have multiple indexes (allowing lookup by multiple keys). Different indexes can use different algorithms (e.g. hash vs. tree) and have different uniqueness constraints.
- The properties on which a Table is indexed need not be explicit fields -- they can be computed from the table's row type.
- Tables use less memory and make fewer allocations than STL, because rows are stored in a contiguous array.
- The hash indexing implementation uses linear probing rather than chaining, which again means far fewer allocations and more cache-friendliness.
- The tree indexing implementation uses B-trees optimized for cache line size, whereas STL uses cache-unfriendly and allocation-heavy red-black binary trees. (However, STL trees are overall more cache-friendly; see below.)
- Most of the b-tree implementation is not templated. This reduces code bloat, at the cost of some performance due to virtual calls.

On an ad hoc benchmark on large tables, the hash index implementation appears to outperform libc++'s `std::unordered_set` by ~60%. However, libc++'s `std::set` still outperforms the B-tree index by ~70%. It looks like the B-tree implementation suffers in part from the fact that keys are not stored inline in the tree nodes, forcing extra memory indirections. This is a price we pay for lower memory usage overall, and the ability to have multiple indexes on one table. The b-tree implementation also suffers somewhat from not being 100% templates, compared to STL, but I think this is a reasonable trade-off. The most performance-critical use cases will use hash indexes anyway.

c++/src/kj/common.h

@@ -530,8 +530,13 @@ template <typename T, size_t s>
 inline constexpr size_t size(T (&arr)[s]) { return s; }
 template <typename T>
 inline constexpr size_t size(T&& arr) { return arr.size(); }
+template <typename T, typename U, size_t s>
+inline constexpr size_t size(U (T::*arr)[s]) { return s; }
 // Returns the size of the parameter, whether the parameter is a regular C array or a container
 // with a `.size()` method.
+//
+// Can also be invoked on a pointer-to-member-array to get the declared size of that array,
+// without having an instance of the containing type. E.g.: kj::size(&MyType::someArray)
 
 class MaxValue_ {
 private:

c++/src/kj/function.h

@@ -213,7 +213,8 @@ public:
 
     // All instances of Wrapper<Func> are two pointers in size: a vtable, and a Func&. So if we
     // allocate space for two pointers, we can construct a Wrapper<Func> in it!
-    static_assert(sizeof(WrapperType) == sizeof(space));
+    static_assert(sizeof(WrapperType) == sizeof(space),
+        "expected WrapperType to be two pointers");
 
     // Even if `func` is an rvalue reference, it's OK to use it as an lvalue here, because
     // FunctionParam is used strictly for parameters. If we captured a temporary, we know that

c++/src/kj/table-test.c++

+// Copyright (c) 2018 Kenton Varda and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "table.h"
+#include <kj/test.h>
+
+namespace kj {
+namespace _ {
+
+KJ_TEST("_::tryReserveSize() works") {
+  {
+    Vector<int> vec;
+    tryReserveSize(vec, "foo"_kj);
+    KJ_EXPECT(vec.capacity() == 3);
+  }
+  {
+    Vector<int> vec;
+    tryReserveSize(vec, 123);
+    KJ_EXPECT(vec.capacity() == 0);
+  }
+}
+
+class StringHasher {
+public:
+  bool matches(StringPtr a, StringPtr b) const {
+    return a == b;
+  }
+  uint hashCode(StringPtr str) const {
+    // djb hash with xor
+    // TDOO(soon): Use KJ hash lib.
+    size_t result = 5381;
+    for (char c: str) {
+      result = (result * 33) ^ c;
+    }
+    return result;
+  }
+};
+
+KJ_TEST("simple table") {
+  Table<StringPtr, HashIndex<StringHasher>> table;
+
+  KJ_EXPECT(table.find("foo") == nullptr);
+
+  KJ_EXPECT(table.size() == 0);
+  KJ_EXPECT(table.insert("foo") == "foo");
+  KJ_EXPECT(table.size() == 1);
+  KJ_EXPECT(table.insert("bar") == "bar");
+  KJ_EXPECT(table.size() == 2);
+
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("foo")) == "foo");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(table.find("fop") == nullptr);
+  KJ_EXPECT(table.find("baq") == nullptr);
+
+  {
+    StringPtr& ref = table.insert("baz");
+    KJ_EXPECT(ref == "baz");
+    StringPtr& ref2 = KJ_ASSERT_NONNULL(table.find("baz"));
+    KJ_EXPECT(&ref == &ref2);
+  }
+
+  KJ_EXPECT(table.size() == 3);
+
+  {
+    auto iter = table.begin();
+    KJ_EXPECT(*iter++ == "foo");
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(iter == table.end());
+  }
+
+  KJ_EXPECT(table.eraseMatch("foo"));
+  KJ_EXPECT(table.size() == 2);
+  KJ_EXPECT(table.find("foo") == nullptr);
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("baz")) == "baz");
+
+  {
+    auto iter = table.begin();
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(iter == table.end());
+  }
+
+  {
+    auto& row = table.upsert("qux", [&](StringPtr&, StringPtr&&) {
+      KJ_FAIL_ASSERT("shouldn't get here");
+    });
+
+    auto copy = kj::str("qux");
+    table.upsert(StringPtr(copy), [&](StringPtr& existing, StringPtr&& param) {
+      KJ_EXPECT(param.begin() == copy.begin());
+      KJ_EXPECT(&existing == &row);
+    });
+
+    auto& found = KJ_ASSERT_NONNULL(table.find("qux"));
+    KJ_EXPECT(&found == &row);
+  }
+
+  StringPtr STRS[] = { "corge"_kj, "grault"_kj, "garply"_kj };
+  table.insertAll(ArrayPtr<StringPtr>(STRS));
+  KJ_EXPECT(table.size() == 6);
+  KJ_EXPECT(table.find("corge") != nullptr);
+  KJ_EXPECT(table.find("grault") != nullptr);
+  KJ_EXPECT(table.find("garply") != nullptr);
+
+  KJ_EXPECT_THROW_MESSAGE("inserted row already exists in table", table.insert("bar"));
+
+  KJ_EXPECT(table.size() == 6);
+
+  KJ_EXPECT(table.insert("baa") == "baa");
+
+  KJ_EXPECT(table.eraseAll([](StringPtr s) { return s.startsWith("ba"); }) == 3);
+  KJ_EXPECT(table.size() == 4);
+
+  {
+    auto iter = table.begin();
+    KJ_EXPECT(*iter++ == "garply");
+    KJ_EXPECT(*iter++ == "grault");
+    KJ_EXPECT(*iter++ == "qux");
+    KJ_EXPECT(*iter++ == "corge");
+    KJ_EXPECT(iter == table.end());
+  }
+}
+
+class BadHasher {
+  // String hash that always returns the same hash code. This should not affect correctness, only
+  // performance.
+public:
+  bool matches(StringPtr a, StringPtr b) const {
+    return a == b;
+  }
+  uint hashCode(StringPtr str) const {
+    return 1234;
+  }
+};
+
+KJ_TEST("hash tables when hash is always same") {
+  Table<StringPtr, HashIndex<BadHasher>> table;
+
+  KJ_EXPECT(table.size() == 0);
+  KJ_EXPECT(table.insert("foo") == "foo");
+  KJ_EXPECT(table.size() == 1);
+  KJ_EXPECT(table.insert("bar") == "bar");
+  KJ_EXPECT(table.size() == 2);
+
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("foo")) == "foo");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(table.find("fop") == nullptr);
+  KJ_EXPECT(table.find("baq") == nullptr);
+
+  {
+    StringPtr& ref = table.insert("baz");
+    KJ_EXPECT(ref == "baz");
+    StringPtr& ref2 = KJ_ASSERT_NONNULL(table.find("baz"));
+    KJ_EXPECT(&ref == &ref2);
+  }
+
+  KJ_EXPECT(table.size() == 3);
+
+  {
+    auto iter = table.begin();
+    KJ_EXPECT(*iter++ == "foo");
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(iter == table.end());
+  }
+
+  KJ_EXPECT(table.eraseMatch("foo"));
+  KJ_EXPECT(table.size() == 2);
+  KJ_EXPECT(table.find("foo") == nullptr);
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("baz")) == "baz");
+
+  {
+    auto iter = table.begin();
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(iter == table.end());
+  }
+
+  {
+    auto& row = table.upsert("qux", [&](StringPtr&, StringPtr&&) {
+      KJ_FAIL_ASSERT("shouldn't get here");
+    });
+
+    auto copy = kj::str("qux");
+    table.upsert(StringPtr(copy), [&](StringPtr& existing, StringPtr&& param) {
+      KJ_EXPECT(param.begin() == copy.begin());
+      KJ_EXPECT(&existing == &row);
+    });
+
+    auto& found = KJ_ASSERT_NONNULL(table.find("qux"));
+    KJ_EXPECT(&found == &row);
+  }
+
+  StringPtr STRS[] = { "corge"_kj, "grault"_kj, "garply"_kj };
+  table.insertAll(ArrayPtr<StringPtr>(STRS));
+  KJ_EXPECT(table.size() == 6);
+  KJ_EXPECT(table.find("corge") != nullptr);
+  KJ_EXPECT(table.find("grault") != nullptr);
+  KJ_EXPECT(table.find("garply") != nullptr);
+
+  KJ_EXPECT_THROW_MESSAGE("inserted row already exists in table", table.insert("bar"));
+}
+
+struct SiPair {
+  kj::StringPtr str;
+  uint i;
+
+  inline bool operator==(SiPair other) const {
+    return str == other.str && i == other.i;
+  }
+};
+
+class SiPairStringHasher {
+public:
+  bool matches(SiPair a, SiPair b) const {
+    return a.str == b.str;
+  }
+  uint hashCode(SiPair pair) const {
+    return inner.hashCode(pair.str);
+  }
+
+  bool matches(SiPair a, StringPtr b) const {
+    return a.str == b;
+  }
+  uint hashCode(StringPtr str) const {
+    return inner.hashCode(str);
+  }
+
+private:
+  StringHasher inner;
+};
+
+class SiPairIntHasher {
+public:
+  bool matches(SiPair a, SiPair b) const {
+    return a.i == b.i;
+  }
+  uint hashCode(SiPair pair) const {
+    return pair.i;
+  }
+
+  bool matches(SiPair a, uint b) const {
+    return a.i == b;
+  }
+  uint hashCode(uint i) const {
+    return i;
+  }
+};
+
+KJ_TEST("double-index table") {
+  Table<SiPair, HashIndex<SiPairStringHasher>, HashIndex<SiPairIntHasher>> table;
+
+  KJ_EXPECT(table.size() == 0);
+  KJ_EXPECT(table.insert({"foo", 123}) == (SiPair {"foo", 123}));
+  KJ_EXPECT(table.size() == 1);
+  KJ_EXPECT(table.insert({"bar", 456}) == (SiPair {"bar", 456}));
+  KJ_EXPECT(table.size() == 2);
+
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<HashIndex<SiPairStringHasher>>("foo")) ==
+            (SiPair {"foo", 123}));
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<HashIndex<SiPairIntHasher>>(123)) ==
+            (SiPair {"foo", 123}));
+
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<0>("foo")) == (SiPair {"foo", 123}));
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<1>(123)) == (SiPair {"foo", 123}));
+
+  KJ_EXPECT_THROW_MESSAGE("inserted row already exists in table", table.insert({"foo", 111}));
+  KJ_EXPECT_THROW_MESSAGE("inserted row already exists in table", table.insert({"qux", 123}));
+
+  KJ_EXPECT(table.size() == 2);
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<0>("foo")) == (SiPair {"foo", 123}));
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find<1>(123)) == (SiPair {"foo", 123}));
+}
+
+class UintHasher {
+public:
+  bool matches(uint a, uint b) const {
+    return a == b;
+  }
+  uint hashCode(uint i) const {
+    return i;
+  }
+};
+
+KJ_TEST("large hash table") {
+  constexpr uint SOME_PRIME = 6143;
+  constexpr uint STEP[] = {1, 2, 4, 7, 43, 127};
+
+  for (auto step: STEP) {
+    KJ_CONTEXT(step);
+    Table<uint, HashIndex<UintHasher>> table;
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      uint j = (i * step) % SOME_PRIME;
+      table.insert(j * 5 + 123);
+    }
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      uint value = KJ_ASSERT_NONNULL(table.find(i * 5 + 123));
+      KJ_ASSERT(value == i * 5 + 123);
+      KJ_ASSERT(table.find(i * 5 + 122) == nullptr);
+      KJ_ASSERT(table.find(i * 5 + 124) == nullptr);
+    }
+
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      if (i % 2 == 0 || i % 7 == 0) {
+        table.erase(KJ_ASSERT_NONNULL(table.find(i * 5 + 123)));
+      }
+    }
+
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      if (i % 2 == 0 || i % 7 == 0) {
+        // erased
+        KJ_ASSERT(table.find(i * 5 + 123) == nullptr);
+      } else {
+        uint value = KJ_ASSERT_NONNULL(table.find(i * 5 + 123));
+        KJ_ASSERT(value == i * 5 + 123);
+      }
+    }
+  }
+}
+
+// =======================================================================================
+
+KJ_TEST("B-tree internals") {
+  {
+    BTreeImpl::Leaf leaf;
+    memset(&leaf, 0, sizeof(leaf));
+
+    for (auto i: kj::indices(leaf.rows)) {
+      KJ_CONTEXT(i);
+
+      KJ_EXPECT(leaf.size() == i);
+
+      if (i < kj::size(leaf.rows) / 2) {
+#ifdef KJ_DEBUG
+        KJ_EXPECT_THROW(FAILED, leaf.isHalfFull());
+#endif
+        KJ_EXPECT(!leaf.isMostlyFull());
+      }
+
+      if (i == kj::size(leaf.rows) / 2) {
+        KJ_EXPECT(leaf.isHalfFull());
+        KJ_EXPECT(!leaf.isMostlyFull());
+      }
+
+      if (i > kj::size(leaf.rows) / 2) {
+        KJ_EXPECT(!leaf.isHalfFull());
+        KJ_EXPECT(leaf.isMostlyFull());
+      }
+
+      if (i == kj::size(leaf.rows)) {
+        KJ_EXPECT(leaf.isFull());
+      } else {
+        KJ_EXPECT(!leaf.isFull());
+      }
+
+      leaf.rows[i] = 1;
+    }
+    KJ_EXPECT(leaf.size() == kj::size(leaf.rows));
+  }
+
+  {
+    BTreeImpl::Parent parent;
+    memset(&parent, 0, sizeof(parent));
+
+    for (auto i: kj::indices(parent.keys)) {
+      KJ_CONTEXT(i);
+
+      KJ_EXPECT(parent.keyCount() == i);
+
+      if (i < kj::size(parent.keys) / 2) {
+#ifdef KJ_DEBUG
+        KJ_EXPECT_THROW(FAILED, parent.isHalfFull());
+#endif
+        KJ_EXPECT(!parent.isMostlyFull());
+      }
+
+      if (i == kj::size(parent.keys) / 2) {
+        KJ_EXPECT(parent.isHalfFull());
+        KJ_EXPECT(!parent.isMostlyFull());
+      }
+
+      if (i > kj::size(parent.keys) / 2) {
+        KJ_EXPECT(!parent.isHalfFull());
+        KJ_EXPECT(parent.isMostlyFull());
+      }
+
+      if (i == kj::size(parent.keys)) {
+        KJ_EXPECT(parent.isFull());
+      } else {
+        KJ_EXPECT(!parent.isFull());
+      }
+
+      parent.keys[i] = 1;
+    }
+    KJ_EXPECT(parent.keyCount() == kj::size(parent.keys));
+  }
+}
+
+class StringCompare {
+public:
+  bool isBefore(StringPtr a, StringPtr b) const {
+    return a < b;
+  }
+  bool matches(StringPtr a, StringPtr b) const {
+    return a == b;
+  }
+};
+
+KJ_TEST("simple tree table") {
+  Table<StringPtr, TreeIndex<StringCompare>> table;
+
+  KJ_EXPECT(table.find("foo") == nullptr);
+
+  KJ_EXPECT(table.size() == 0);
+  KJ_EXPECT(table.insert("foo") == "foo");
+  KJ_EXPECT(table.size() == 1);
+  KJ_EXPECT(table.insert("bar") == "bar");
+  KJ_EXPECT(table.size() == 2);
+
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("foo")) == "foo");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(table.find("fop") == nullptr);
+  KJ_EXPECT(table.find("baq") == nullptr);
+
+  {
+    StringPtr& ref = table.insert("baz");
+    KJ_EXPECT(ref == "baz");
+    StringPtr& ref2 = KJ_ASSERT_NONNULL(table.find("baz"));
+    KJ_EXPECT(&ref == &ref2);
+  }
+
+  KJ_EXPECT(table.size() == 3);
+
+  {
+    auto range = table.ordered();
+    auto iter = range.begin();
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(*iter++ == "foo");
+    KJ_EXPECT(iter == range.end());
+  }
+
+  KJ_EXPECT(table.eraseMatch("foo"));
+  KJ_EXPECT(table.size() == 2);
+  KJ_EXPECT(table.find("foo") == nullptr);
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("bar")) == "bar");
+  KJ_EXPECT(KJ_ASSERT_NONNULL(table.find("baz")) == "baz");
+
+  {
+    auto range = table.ordered();
+    auto iter = range.begin();
+    KJ_EXPECT(*iter++ == "bar");
+    KJ_EXPECT(*iter++ == "baz");
+    KJ_EXPECT(iter == range.end());
+  }
+
+  {
+    auto& row = table.upsert("qux", [&](StringPtr&, StringPtr&&) {
+      KJ_FAIL_ASSERT("shouldn't get here");
+    });
+
+    auto copy = kj::str("qux");
+    table.upsert(StringPtr(copy), [&](StringPtr& existing, StringPtr&& param) {
+      KJ_EXPECT(param.begin() == copy.begin());
+      KJ_EXPECT(&existing == &row);
+    });
+
+    auto& found = KJ_ASSERT_NONNULL(table.find("qux"));
+    KJ_EXPECT(&found == &row);
+  }
+
+  StringPtr STRS[] = { "corge"_kj, "grault"_kj, "garply"_kj };
+  table.insertAll(ArrayPtr<StringPtr>(STRS));
+  KJ_EXPECT(table.size() == 6);
+  KJ_EXPECT(table.find("corge") != nullptr);
+  KJ_EXPECT(table.find("grault") != nullptr);
+  KJ_EXPECT(table.find("garply") != nullptr);
+
+  KJ_EXPECT_THROW_MESSAGE("inserted row already exists in table", table.insert("bar"));
+
+  KJ_EXPECT(table.size() == 6);
+
+  KJ_EXPECT(table.insert("baa") == "baa");
+
+  KJ_EXPECT(table.eraseAll([](StringPtr s) { return s.startsWith("ba"); }) == 3);
+  KJ_EXPECT(table.size() == 4);
+
+  {
+    auto range = table.ordered();
+    auto iter = range.begin();
+    KJ_EXPECT(*iter++ == "corge");
+    KJ_EXPECT(*iter++ == "garply");
+    KJ_EXPECT(*iter++ == "grault");
+    KJ_EXPECT(*iter++ == "qux");
+    KJ_EXPECT(iter == range.end());
+  }
+
+  {
+    auto range = table.range("foo", "har");
+    auto iter = range.begin();
+    KJ_EXPECT(*iter++ == "garply");
+    KJ_EXPECT(*iter++ == "grault");
+    KJ_EXPECT(iter == range.end());
+  }
+
+  {
+    auto range = table.range("garply", "grault");
+    auto iter = range.begin();
+    KJ_EXPECT(*iter++ == "garply");
+    KJ_EXPECT(iter == range.end());
+  }
+}
+
+class UintCompare {
+public:
+  bool isBefore(uint a, uint b) const {
+    return a < b;
+  }
+  bool matches(uint a, uint b) const {
+    return a == b;
+  }
+};
+
+KJ_TEST("large tree table") {
+  constexpr uint SOME_PRIME = 6143;
+  constexpr uint STEP[] = {1, 2, 4, 7, 43, 127};
+
+  for (auto step: STEP) {
+    KJ_CONTEXT(step);
+    Table<uint, TreeIndex<UintCompare>> table;
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      uint j = (i * step) % SOME_PRIME;
+      table.insert(j * 5 + 123);
+    }
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      uint value = KJ_ASSERT_NONNULL(table.find(i * 5 + 123));
+      KJ_ASSERT(value == i * 5 + 123);
+      KJ_ASSERT(table.find(i * 5 + 122) == nullptr);
+      KJ_ASSERT(table.find(i * 5 + 124) == nullptr);
+    }
+    table.verify();
+
+    {
+      auto range = table.ordered();
+      auto iter = range.begin();
+      for (uint i: kj::zeroTo(SOME_PRIME)) {
+        KJ_ASSERT(*iter++ == i * 5 + 123);
+      }
+      KJ_ASSERT(iter == range.end());
+    }
+
+    for (uint i: kj::zeroTo(SOME_PRIME)) {
+      KJ_CONTEXT(i);
+      if (i % 2 == 0 || i % 7 == 0) {
+        table.erase(KJ_ASSERT_NONNULL(table.find(i * 5 + 123), i));
+        table.verify();
+      }
+    }
+
+    {
+      auto range = table.ordered();
+      auto iter = range.begin();
+      for (uint i: kj::zeroTo(SOME_PRIME)) {
+        if (i % 2 == 0 || i % 7 == 0) {
+          // erased
+          KJ_ASSERT(table.find(i * 5 + 123) == nullptr);
+        } else {
+          uint value = KJ_ASSERT_NONNULL(table.find(i * 5 + 123));
+          KJ_ASSERT(value == i * 5 + 123);
+          KJ_ASSERT(*iter++ == i * 5 + 123);
+        }
+      }
+      KJ_ASSERT(iter == range.end());
+    }
+  }
+}
+
+}  // namespace kj
+}  // namespace _

c++/src/kj/table.c++

+// Copyright (c) 2018 Kenton Varda and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "table.h"
+#include "debug.h"
+#include <stdlib.h>
+
+namespace kj {
+namespace _ {
+
+static constexpr uint lg(uint value) {
+  // Compute floor(log2(value)).
+  return sizeof(uint) * 8 - 1 - __builtin_clz(value);
+}
+
+void throwDuplicateTableRow() {
+  KJ_FAIL_REQUIRE("inserted row already exists in table");
+}
+
+void logHashTableInconsistency() {
+  KJ_LOG(ERROR,
+      "HashIndex detected hash table inconsistency. This can happen if you create a kj::Table "
+      "with a hash index and you modify the rows in the table post-indexing in a way that would "
+      "change their hash. This is a serious bug which will lead to undefined behavior."
+      "\nstack: ", kj::getStackTrace());
+}
+
+// List of primes where each element is roughly double the previous.  Obtained
+// from:
+//   http://planetmath.org/goodhashtableprimes
+// Primes < 53 were added to ensure that small tables don't allocate excessive memory.
+static const size_t PRIMES[] = {
+           1,  // 2^ 0 = 1
+           3,  // 2^ 1 = 2
+           5,  // 2^ 2 = 4
+          11,  // 2^ 3 = 8
+          23,  // 2^ 4 = 16
+          53,  // 2^ 5 = 32
+          97,  // 2^ 6 = 64
+         193,  // 2^ 7 = 128
+         389,  // 2^ 8 = 256
+         769,  // 2^ 9 = 512
+        1543,  // 2^10 = 1024
+        3079,  // 2^11 = 2048
+        6151,  // 2^12 = 4096
+       12289,  // 2^13 = 8192
+       24593,  // 2^14 = 16384
+       49157,  // 2^15 = 32768
+       98317,  // 2^16 = 65536
+      196613,  // 2^17 = 131072
+      393241,  // 2^18 = 262144
+      786433,  // 2^19 = 524288
+     1572869,  // 2^20 = 1048576
+     3145739,  // 2^21 = 2097152
+     6291469,  // 2^22 = 4194304
+    12582917,  // 2^23 = 8388608
+    25165843,  // 2^24 = 16777216
+    50331653,  // 2^25 = 33554432
+   100663319,  // 2^26 = 67108864
+   201326611,  // 2^27 = 134217728
+   402653189,  // 2^28 = 268435456
+   805306457,  // 2^29 = 536870912
+  1610612741,  // 2^30 = 1073741824
+};
+
+size_t chooseHashTableSize(uint size) {
+  if (size == 0) return 0;
+
+  // Add 1 to compensate for the floor() above, then look up the best prime bucket size for that
+  // target size.
+  return PRIMES[lg(size) + 1];
+}
+
+kj::Array<HashBucket> rehash(kj::ArrayPtr<const HashBucket> oldBuckets, size_t targetSize) {
+  // Rehash the whole table.
+  // The element at `invalidPos` will be ignored.
+  // The element at `replacePos` will be recorded as if it were located at `invalidPos`.
+
+  KJ_REQUIRE(targetSize < (1 << 30), "hash table has reached maximum size");
+
+  size_t size = chooseHashTableSize(targetSize);
+
+  if (size < oldBuckets.size()) {
+    size = oldBuckets.size();
+  }
+
+  auto newBuckets = kj::heapArray<HashBucket>(size);
+  memset(newBuckets.begin(), 0, sizeof(HashBucket) * size);
+
+  for (auto& oldBucket: oldBuckets) {
+    if (oldBucket.isOccupied()) {
+      for (uint i = oldBucket.hash % newBuckets.size();; i = probeHash(newBuckets, i)) {
+        auto& newBucket = newBuckets[i];
+        if (newBucket.isEmpty()) {
+          newBucket = oldBucket;
+          break;
+        }
+      }
+    }
+  }
+
+  return newBuckets;
+}
+
+// =======================================================================================
+// BTree
+
+BTreeImpl::BTreeImpl()
+    : tree(const_cast<NodeUnion*>(&EMPTY_NODE)),
+      treeCapacity(1),
+      height(0),
+      freelistHead(1),
+      freelistSize(0),
+      beginLeaf(0),
+      endLeaf(0) {}
+BTreeImpl::~BTreeImpl() noexcept(false) {
+  if (tree != &EMPTY_NODE) {
+    ::free(tree);
+  }
+}
+
+const BTreeImpl::NodeUnion BTreeImpl::EMPTY_NODE = {{{0, {0}}}};
+
+void BTreeImpl::verify(size_t size, FunctionParam<bool(uint, uint)> f) {
+  KJ_ASSERT(verifyNode(size, f, 0, height, nullptr) == size);
+}
+size_t BTreeImpl::verifyNode(size_t size, FunctionParam<bool(uint, uint)>& f,
+                             uint pos, uint height, MaybeUint maxRow) {
+  if (height > 0) {
+    auto& parent = tree[pos].parent;
+
+    auto n = parent.keyCount();
+    size_t total = 0;
+    for (auto i: kj::zeroTo(n)) {
+      KJ_ASSERT(*parent.keys[i] < size);
+      total += verifyNode(size, f, parent.children[i], height - 1, parent.keys[i]);
+      KJ_ASSERT(i + 1 == n || f(*parent.keys[i], *parent.keys[i + 1]));
+    }
+    total += verifyNode(size, f, parent.children[n], height - 1, maxRow);
+    KJ_ASSERT(maxRow == nullptr || f(*parent.keys[n-1], *maxRow));
+    return total;
+  } else {
+    auto& leaf = tree[pos].leaf;
+    auto n = leaf.size();
+    for (auto i: kj::zeroTo(n)) {
+      KJ_ASSERT(*leaf.rows[i] < size);
+      if (i + 1 < n) {
+        KJ_ASSERT(f(*leaf.rows[i], *leaf.rows[i + 1]));
+      } else {
+        KJ_ASSERT(maxRow == nullptr || leaf.rows[n-1] == maxRow);
+      }
+    }
+    return n;
+  }
+}
+
+void BTreeImpl::logInconsitency() const {
+  KJ_LOG(ERROR,
+      "BTreeIndex detected tree state inconsistency. This can happen if you create a kj::Table "
+      "with a b-tree index and you modify the rows in the table post-indexing in a way that would "
+      "change their ordering. This is a serious bug which will lead to undefined behavior."
+      "\nstack: ", kj::getStackTrace());
+}
+
+void BTreeImpl::reserve(size_t size) {
+  KJ_REQUIRE(size < (1u << 31), "b-tree has reached maximum size");
+
+  // Calculate the worst-case number of leaves to cover the size, given that a leaf is always at
+  // least half-full. (Note that it's correct for this calculation to round down, not up: The
+  // remainder will necessarily be distributed among the non-full leaves, rather than creating a
+  // new leaf, because if it went into a new leaf, that leaf would be less than half-full.)
+  uint leaves = size / (kj::size(&Leaf::rows) / 2);
+
+  // Calculate the worst-case number of parents to cover the leaves, given that a parent is always
+  // at least half-full. Since the parents form a tree with branching factor B, the size of the
+  // tree is N/B + N/B^2 + N/B^3 + N/B^4 + ... = N / (B - 1). Math.
+  constexpr uint branchingFactor = kj::size(&Parent::children) / 2;
+  uint parents = leaves / (branchingFactor - 1);
+
+  // Height is log-base-branching-factor of leaves, plus 1 for the root node.
+  uint height = lg(leaves | 1) / lg(branchingFactor) + 1;
+
+  size_t newSize = leaves +
+      parents + 1 +  // + 1 for the root
+      height + 2;    // minimum freelist size needed by insert()
+
+  if (treeCapacity < newSize) {
+    growTree(newSize);
+  }
+}
+
+void BTreeImpl::clear() {
+  if (tree != &EMPTY_NODE) {
+    azero(tree, treeCapacity);
+    height = 0;
+    freelistHead = 1;
+    freelistSize = treeCapacity;
+    beginLeaf = 0;
+    endLeaf = 0;
+  }
+}
+
+void BTreeImpl::growTree(uint minCapacity) {
+  uint newCapacity = kj::max(kj::max(minCapacity, treeCapacity * 2), 4);
+  freelistSize += newCapacity - treeCapacity;
+  NodeUnion* newTree = reinterpret_cast<NodeUnion*>(
+      aligned_alloc(sizeof(BTreeImpl::NodeUnion), newCapacity * sizeof(BTreeImpl::NodeUnion)));
+  KJ_ASSERT(newTree != nullptr, "memory allocation failed", newCapacity);
+  acopy(newTree, tree, treeCapacity);
+  azero(newTree + treeCapacity, newCapacity - treeCapacity);
+  if (tree != &EMPTY_NODE) ::free(tree);
+  tree = newTree;
+  treeCapacity = newCapacity;
+}
+
+BTreeImpl::Iterator BTreeImpl::search(const SearchKey& searchKey) const {
+  // Find the "first" row number (in sorted order) for which predicate(rowNumber) returns true.
+
+  uint pos = 0;
+
+  for (auto i KJ_UNUSED: zeroTo(height)) {
+    auto& parent = tree[pos].parent;
+    pos = parent.children[searchKey.search(parent)];
+  }
+
+  {
+    auto& leaf = tree[pos].leaf;
+    return { tree, &leaf, searchKey.search(leaf) };
+  }
+}
+
+template <typename T>
+struct BTreeImpl::AllocResult {
+  uint index;
+  T& node;
+};
+
+template <typename T>
+inline BTreeImpl::AllocResult<T> BTreeImpl::alloc() {
+  // Allocate a new item from the freelist. Guaranteed to be zero'd except for the first member.
+  uint i = freelistHead;
+  NodeUnion* ptr = &tree[i];
+  freelistHead = i + 1 + ptr->freelist.nextOffset;
+  --freelistSize;
+  return { i, *ptr };
+}
+
+inline void BTreeImpl::free(uint pos) {
+  // Add the given node to the freelist.
+
+  // HACK: This is typically called on a node immediately after copying its contents away, but the
+  //   pointer used to copy it away may be a different pointer pointing to a different union member
+  //   which the compiler may not recgonize as aliasing with this object. Just to be extra-safe,
+  //   insert a compiler barrier.
+  compilerBarrier();
+
+  auto& node = tree[pos];
+  node.freelist.nextOffset = freelistHead - pos - 1;
+  azero(node.freelist.zero, kj::size(node.freelist.zero));
+  freelistHead = pos;
+  ++freelistSize;
+}
+
+BTreeImpl::Iterator BTreeImpl::insert(const SearchKey& searchKey) {
+  // Like search() but ensures that there is room in the leaf node to insert a new row.
+
+  // If we split the root node it will generate two new nodes. If we split any other node in the
+  // path it will generate one new node. `height` doesn't count leaf nodes, but we can equivalently
+  // think of it as not counting the root node, so in the worst case we may allocate height + 2
+  // new nodes.
+  //
+  // (Also note that if the tree is currently empty, then `tree` points to a dummy root node in
+  // read-only memory. We definitely need to allocate a real tree node array in this case, and
+  // we'll start out allocating space for four nodes, which will be all we need up to 28 rows.)
+  if (freelistSize < height + 2) {
+    if (height > 0 && !tree[0].parent.isFull() && freelistSize >= height) {
+      // Slight optimization: The root node is not full, so we're definitely not going to split it.
+      // That means that the maximum allocations we might do is equal to `height`, not
+      // `height + 2`, and we have that much space, so no need to grow yet.
+      //
+      // This optimization is particularly important for small trees, e.g. when treeCapacity is 4
+      // and the tree so far consists of a root and two children, we definitely don't need to grow
+      // the tree yet.
+    } else {
+      growTree();
+
+      if (freelistHead == 0) {
+        // We have no root yet. Allocate one.
+        KJ_ASSERT(alloc<Parent>().index == 0);
+      }
+    }
+  }
+
+  uint pos = 0;
+
+  // Track grandparent node and child index within grandparent.
+  Parent* parent = nullptr;
+  uint indexInParent = 0;
+
+  for (auto i KJ_UNUSED: zeroTo(height)) {
+    Parent& node = insertHelper(searchKey, tree[pos].parent, parent, indexInParent, pos);
+
+    parent = &node;
+    indexInParent = searchKey.search(node);
+    pos = node.children[indexInParent];
+  }
+
+  {
+    Leaf& leaf = insertHelper(searchKey, tree[pos].leaf, parent, indexInParent, pos);
+
+    // Fun fact: Unlike erase(), there's no need to climb back up the tree modifying keys, because
+    // either the newly-inserted node will not be the last in the leaf (and thus parent keys aren't
+    // modified), or the leaf is the last leaf in the tree (and thus there's no parent key to
+    // modify).
+
+    return { tree, &leaf, searchKey.search(leaf) };
+  }
+}
+
+template <typename Node>
+Node& BTreeImpl::insertHelper(const SearchKey& searchKey,
+    Node& node, Parent* parent, uint indexInParent, uint pos) {
+  if (node.isFull()) {
+    // This node is full. Need to split.
+    if (parent == nullptr) {
+      // This is the root node. We need to split into two nodes and create a new root.
+      auto n1 = alloc<Node>();
+      auto n2 = alloc<Node>();
+
+      uint pivot = split(n2.node, n2.index, node, pos);
+      move(n1.node, n1.index, node);
+
+      // Rewrite root to have the two children.
+      tree[0].parent.initRoot(pivot, n1.index, n2.index);
+
+      // Increased height.
+      ++height;
+
+      // Decide which new branch has our search key.
+      if (searchKey.isAfter(pivot)) {
+        // the right one
+        return n2.node;
+      } else {
+        // the left one
+        return n1.node;
+      }
+    } else {
+      // This is a non-root parent node. We need to split it into two and insert the new node
+      // into the grandparent.
+      auto n = alloc<Node>();
+      uint pivot = split(n.node, n.index, node, pos);
+
+      // Insert new child into grandparent.
+      parent->insertAfter(indexInParent, pivot, n.index);
+
+      // Decide which new branch has our search key.
+      if (searchKey.isAfter(pivot)) {
+        // the new one, which is right of the original
+        return n.node;
+      } else {
+        // the original one, which is left of the new one
+        return node;
+      }
+    }
+  } else {
+    // No split needed.
+    return node;
+  }
+}
+
+void BTreeImpl::erase(uint row, const SearchKey& searchKey) {
+  // Erase the given row number from the tree. predicate() returns true for the given row and all
+  // rows after it.
+
+  uint pos = 0;
+
+  // Track grandparent node and child index within grandparent.
+  Parent* parent = nullptr;
+  uint indexInParent = 0;
+
+  MaybeUint* fixup = nullptr;
+
+  for (auto i KJ_UNUSED: zeroTo(height)) {
+    Parent& node = eraseHelper(tree[pos].parent, parent, indexInParent, pos, fixup);
+
+    parent = &node;
+    indexInParent = searchKey.search(node);
+    pos = node.children[indexInParent];
+
+    if (indexInParent < kj::size(node.keys) && node.keys[indexInParent] == row) {
+      // Oh look, the row is a key in this node! We'll need to come back and fix this up later.
+      // Note that any particular row can only appear as *one* key value anywhere in the tree, so
+      // we only need one fixup pointer, which is nice.
+      MaybeUint* newFixup = &node.keys[indexInParent];
+      if (fixup == newFixup) {
+        // The fixup pointer was already set while processing a parent node, and then a merge or
+        // rotate caused it to be moved, but the fixup pointer was updated... so it's already set
+        // to point at the slot we wanted it to point to, so nothing to see here.
+      } else {
+        KJ_DASSERT(fixup == nullptr);
+        fixup = newFixup;
+      }
+    }
+  }
+
+  {
+    Leaf& leaf = eraseHelper(tree[pos].leaf, parent, indexInParent, pos, fixup);
+
+    uint r = searchKey.search(leaf);
+    if (leaf.rows[r] == row) {
+      leaf.erase(r);
+
+      if (fixup != nullptr) {
+        // There's a key in a parent node that needs fixup. This is only possible if the removed
+        // node is the last in its leaf.
+        KJ_DASSERT(leaf.rows[r] == nullptr);
+        KJ_DASSERT(r > 0);  // non-root nodes must be at least half full so this can't be item 0
+        KJ_DASSERT(*fixup == row);
+        *fixup = leaf.rows[r - 1];
+      }
+    } else {
+      logInconsitency();
+    }
+  }
+}
+
+template <typename Node>
+Node& BTreeImpl::eraseHelper(
+    Node& node, Parent* parent, uint indexInParent, uint pos, MaybeUint*& fixup) {
+  if (parent != nullptr && !node.isMostlyFull()) {
+    // This is not the root, but it's only half-full. Rebalance.
+    KJ_DASSERT(node.isHalfFull());
+
+    if (indexInParent > 0) {
+      // There's a sibling to the left.
+      uint sibPos = parent->children[indexInParent - 1];
+      Node& sib = tree[sibPos];
+      if (sib.isMostlyFull()) {
+        // Left sibling is more than half full. Steal one member.
+        rotateRight(sib, node, *parent, indexInParent - 1);
+        return node;
+      } else {
+        // Left sibling is half full, too. Merge.
+        KJ_ASSERT(sib.isHalfFull());
+        merge(sib, sibPos, *parent->keys[indexInParent - 1], node);
+        parent->eraseAfter(indexInParent - 1);
+        free(pos);
+        if (fixup == &parent->keys[indexInParent]) --fixup;
+
+        if (parent->keys[0] == nullptr) {
+          // Oh hah, the parent has no keys left. It must be the root. We can eliminate it.
+          KJ_DASSERT(parent == &tree->parent);
+          compilerBarrier();  // don't reorder any writes to parent below here
+          move(tree[0], 0, sib);
+          free(sibPos);
+          --height;
+          return tree[0];
+        } else {
+          return sib;
+        }
+      }
+    } else if (indexInParent < kj::size(&Parent::keys) && parent->keys[indexInParent] != nullptr) {
+      // There's a sibling to the right.
+      uint sibPos = parent->children[indexInParent + 1];
+      Node& sib = tree[sibPos];
+      if (sib.isMostlyFull()) {
+        // Right sibling is more than half full. Steal one member.
+        rotateLeft(node, sib, *parent, indexInParent, fixup);
+        return node;
+      } else {
+        // Right sibling is half full, too. Merge.
+        KJ_ASSERT(sib.isHalfFull());
+        merge(node, pos, *parent->keys[indexInParent], sib);
+        parent->eraseAfter(indexInParent);
+        free(sibPos);
+        if (fixup == &parent->keys[indexInParent]) fixup = nullptr;
+
+        if (parent->keys[0] == nullptr) {
+          // Oh hah, the parent has no keys left. It must be the root. We can eliminate it.
+          KJ_DASSERT(parent == &tree->parent);
+          compilerBarrier();  // don't reorder any writes to parent below here
+          move(tree[0], 0, node);
+          free(pos);
+          --height;
+          return tree[0];
+        } else {
+          return node;
+        }
+      }
+    } else {
+      KJ_FAIL_ASSERT("inconsistent b-tree");
+    }
+  }
+
+  return node;
+}
+
+void BTreeImpl::renumber(uint oldRow, uint newRow, const SearchKey& searchKey) {
+  // Renumber the given row from oldRow to newRow. predicate() returns true for oldRow and all
+  // rows after it. (It will not be called on newRow.)
+
+  uint pos = 0;
+
+  for (auto i KJ_UNUSED: zeroTo(height)) {
+    auto& node = tree[pos].parent;
+    uint indexInParent = searchKey.search(node);
+    pos = node.children[indexInParent];
+    if (node.keys[indexInParent] == oldRow) {
+      node.keys[indexInParent] = newRow;
+    }
+    KJ_DASSERT(pos != 0);
+  }
+
+  {
+    auto& leaf = tree[pos].leaf;
+    uint r = searchKey.search(leaf);
+    if (leaf.rows[r] == oldRow) {
+      leaf.rows[r] = newRow;
+    } else {
+      logInconsitency();
+    }
+  }
+}
+
+uint BTreeImpl::split(Parent& dst, uint dstPos, Parent& src, uint srcPos) {
+  constexpr size_t mid = kj::size(&Parent::keys) / 2;
+  uint pivot = *src.keys[mid];
+  acopy(dst.keys, src.keys + mid + 1, kj::size(&Parent::keys) - mid - 1);
+  azero(src.keys + mid, kj::size(&Parent::keys) - mid);
+  acopy(dst.children, src.children + mid + 1, kj::size(&Parent::children) - mid - 1);
+  azero(src.children + mid + 1, kj::size(&Parent::children) - mid - 1);
+  return pivot;
+}
+
+uint BTreeImpl::split(Leaf& dst, uint dstPos, Leaf& src, uint srcPos) {
+  constexpr size_t mid = kj::size(&Leaf::rows) / 2;
+  uint pivot = *src.rows[mid - 1];
+  acopy(dst.rows, src.rows + mid, kj::size(&Leaf::rows) - mid);
+  azero(src.rows + mid, kj::size(&Leaf::rows) - mid);
+
+  if (src.next == 0) {
+    endLeaf = dstPos;
+  } else {
+    tree[src.next].leaf.prev = dstPos;
+  }
+  dst.next = src.next;
+  dst.prev = srcPos;
+  src.next = dstPos;
+
+  return pivot;
+}
+
+void BTreeImpl::merge(Parent& dst, uint dstPos, uint pivot, Parent& src) {
+  // merge() is only legal if both nodes are half-empty. Meanwhile, B-tree invariants
+  // guarantee that the node can't be more than half-empty, or we would have merged it sooner.
+  // (The root can be more than half-empty, but it is never merged with anything.)
+  KJ_DASSERT(src.isHalfFull());
+  KJ_DASSERT(dst.isHalfFull());
+
+  constexpr size_t mid = kj::size(&Parent::keys)/2;
+  dst.keys[mid] = pivot;
+  acopy(dst.keys + mid + 1, src.keys, mid);
+  acopy(dst.children + mid + 1, src.children, mid + 1);
+}
+
+void BTreeImpl::merge(Leaf& dst, uint dstPos, uint pivot, Leaf& src) {
+  // merge() is only legal if both nodes are half-empty. Meanwhile, B-tree invariants
+  // guarantee that the node can't be more than half-empty, or we would have merged it sooner.
+  // (The root can be more than half-empty, but it is never merged with anything.)
+  KJ_DASSERT(src.isHalfFull());
+  KJ_DASSERT(dst.isHalfFull());
+
+  constexpr size_t mid = kj::size(&Leaf::rows)/2;
+  dst.rows[mid] = pivot;
+  acopy(dst.rows + mid, src.rows, mid);
+
+  dst.next = src.next;
+  if (dst.next == 0) {
+    endLeaf = dstPos;
+  } else {
+    tree[dst.next].leaf.prev = dstPos;
+  }
+}
+
+void BTreeImpl::move(Parent& dst, uint dstPos, Parent& src) {
+  dst = src;
+}
+
+void BTreeImpl::move(Leaf& dst, uint dstPos, Leaf& src) {
+  dst = src;
+  if (src.next == 0) {
+    endLeaf = dstPos;
+  } else {
+    tree[src.next].leaf.prev = dstPos;
+  }
+  if (src.prev == 0) {
+    beginLeaf = dstPos;
+  } else {
+    tree[src.prev].leaf.next = dstPos;
+  }
+}
+
+void BTreeImpl::rotateLeft(
+    Parent& left, Parent& right, Parent& parent, uint indexInParent, MaybeUint*& fixup) {
+  // Steal one item from the right node and move it to the left node.
+
+  // Like mergeFrom(), this is only called on an exactly-half-empty node.
+  KJ_DASSERT(left.isHalfFull());
+  KJ_DASSERT(right.isMostlyFull());
+
+  constexpr size_t mid = kj::size(&Parent::keys)/2;
+  left.keys[mid] = parent.keys[indexInParent];
+  if (fixup == &parent.keys[indexInParent]) fixup = &left.keys[mid];
+  parent.keys[indexInParent] = right.keys[0];
+  left.children[mid + 1] = right.children[0];
+  amove(right.keys, right.keys + 1, kj::size(&Parent::keys) - 1);
+  right.keys[kj::size(&Parent::keys) - 1] = nullptr;
+  amove(right.children, right.children + 1, kj::size(&Parent::children) - 1);
+  right.children[kj::size(&Parent::children) - 1] = 0;
+}
+
+void BTreeImpl::rotateLeft(
+    Leaf& left, Leaf& right, Parent& parent, uint indexInParent, MaybeUint*& fixup) {
+  // Steal one item from the right node and move it to the left node.
+
+  // Like merge(), this is only called on an exactly-half-empty node.
+  KJ_DASSERT(left.isHalfFull());
+  KJ_DASSERT(right.isMostlyFull());
+
+  constexpr size_t mid = kj::size(&Leaf::rows)/2;
+  parent.keys[indexInParent] = left.rows[mid] = right.rows[0];
+  if (fixup == &parent.keys[indexInParent]) fixup = nullptr;
+  amove(right.rows, right.rows + 1, kj::size(&Leaf::rows) - 1);
+  right.rows[kj::size(&Leaf::rows) - 1] = nullptr;
+}
+
+void BTreeImpl::rotateRight(Parent& left, Parent& right, Parent& parent, uint indexInParent) {
+  // Steal one item from the left node and move it to the right node.
+
+  // Like merge(), this is only called on an exactly-half-empty node.
+  KJ_DASSERT(right.isHalfFull());
+  KJ_DASSERT(left.isMostlyFull());
+
+  constexpr size_t mid = kj::size(&Parent::keys)/2;
+  amove(right.keys + 1, right.keys, mid);
+  amove(right.children + 1, right.children, mid + 1);
+
+  uint back = left.keyCount() - 1;
+
+  right.keys[0] = parent.keys[indexInParent];
+  parent.keys[indexInParent] = left.keys[back];
+  right.children[0] = left.children[back + 1];
+  left.keys[back] = nullptr;
+  left.children[back + 1] = 0;
+}
+
+void BTreeImpl::rotateRight(Leaf& left, Leaf& right, Parent& parent, uint indexInParent) {
+  // Steal one item from the left node and move it to the right node.
+
+  // Like mergeFrom(), this is only called on an exactly-half-empty node.
+  KJ_DASSERT(right.isHalfFull());
+  KJ_DASSERT(left.isMostlyFull());
+
+  constexpr size_t mid = kj::size(&Leaf::rows)/2;
+  amove(right.rows + 1, right.rows, mid);
+
+  uint back = left.size() - 1;
+
+  right.rows[0] = left.rows[back];
+  parent.keys[indexInParent] = left.rows[back - 1];
+  left.rows[back] = nullptr;
+}
+
+void BTreeImpl::Parent::initRoot(uint key, uint leftChild, uint rightChild) {
+  // HACK: This is typically called on the root node immediately after copying its contents away,
+  //   but the pointer used to copy it away may be a different pointer pointing to a different
+  //   union member which the compiler may not recgonize as aliasing with this object. Just to
+  //   be extra-safe, insert a compiler barrier.
+  compilerBarrier();
+
+  keys[0] = key;
+  children[0] = leftChild;
+  children[1] = rightChild;
+  azero(keys + 1, kj::size(&Parent::keys) - 1);
+  azero(children + 2, kj::size(&Parent::children) - 2);
+}
+
+void BTreeImpl::Parent::insertAfter(uint i, uint splitKey, uint child) {
+  KJ_IREQUIRE(children[kj::size(&Parent::children) - 1] == 0);  // check not full
+
+  amove(keys + i + 1, keys + i, kj::size(&Parent::keys) - (i + 1));
+  keys[i] = splitKey;
+
+  amove(children + i + 2, children + i + 1, kj::size(&Parent::children) - (i + 2));
+  children[i + 1] = child;
+}
+
+void BTreeImpl::Parent::eraseAfter(uint i) {
+  amove(keys + i, keys + i + 1, kj::size(&Parent::keys) - (i + 1));
+  keys[kj::size(&Parent::keys) - 1] = nullptr;
+  amove(children + i + 1, children + i + 2, kj::size(&Parent::children) - (i + 2));
+  children[kj::size(&Parent::children) - 1] = 0;
+}
+
+}  // namespace _
+}  // namespace kj

c++/src/kj/table.h

+// Copyright (c) 2018 Kenton Varda and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#pragma once
+
+#if defined(__GNUC__) && !KJ_HEADER_WARNINGS
+#pragma GCC system_header
+#endif
+
+#include "common.h"
+#include "tuple.h"
+#include "vector.h"
+#include "function.h"
+
+#if _MSC_VER
+// Need _ReadWriteBarrier
+#if _MSC_VER < 1910
+#include <intrin.h>
+#else
+#include <intrin0.h>
+#endif
+#endif
+
+namespace kj {
+
+namespace _ {  // private
+
+template <typename Inner, typename Mapping>
+class MappedIterable;
+template <typename Row>
+class TableMapping;
+template <typename Row, typename Inner>
+using TableIterable = MappedIterable<Inner, TableMapping<Row>>;
+
+}  // namespace _ (private)
+
+template <typename Row, typename... Indexes>
+class Table {
+  // A table with one or more indexes. This is the KJ alternative to map, set, unordered_map, and
+  // unordered_set.
+  //
+  // Unlike a traditional map, which explicitly stores key/value pairs, a Table simply stores
+  // "rows" of arbitrary type, and then lets the application specify how these should be indexed.
+  // Rows could be indexed on a specific struct field, or they could be indexed based on a computed
+  // property. An index could be hash-based or tree-based. Multiple indexes are supported, making
+  // it easy to construct a "bimap".
+  //
+  // The table has deterministic iteration order based on the sequence of insertions and deletions.
+  // In the case of only insertions, the iteration order is the order of insertion. If deletions
+  // occur, then the current last row is moved to occupy the deleted slot. This determinism is
+  // intended to be reliable for the purpose of testing, etc.
+  //
+  // Each index is a class that looks like:
+  //
+  //   template <typename Key>
+  //   class Index {
+  //   public:
+  //     kj::Maybe<size_t> insert(kj::ArrayPtr<const Row> table, size_t pos);
+  //     // Called to indicate that table[pos] is a newly-added value that needs to be indexed.
+  //     // If this index disallows duplicates and some other matching row already exists, then
+  //     // insert() returns the index of that row -- in this case, the table will roll back the
+  //     // insertion.
+  //     //
+  //     // Insert may throw an exception, in which case the table will roll back insertion.
+  //
+  //     void reserve(size_t size);
+  //     // Called when Table::reserve() is called.
+  //
+  //     void erase(kj::ArrayPtr<const Row> table, size_t pos);
+  //     // Called to indicate that table[pos] is about to be removed, so should be de-indexed.
+  //     //
+  //     // erase() called immediately after insert() must not throw an exception, as it may be
+  //     // called during unwind.
+  //
+  //     void move(kj::ArrayPtr<const Row> table, size_t oldPos, size_t newPos);
+  //     // Called when the value at table[oldPos] is about to be moved to table[newPos].
+  //     // This should never throw; if it does the table may be corrupted.
+  //
+  //     class Iterator;  // Behaves like a C++ iterator over size_t values.
+  //     class Iterable;  // Has begin() and end() methods returning iterators.
+  //
+  //     template <typename... Params>
+  //     Maybe<size_t> find(kj::ArrayPtr<const Row> table, Params&&...) const;
+  //     // Optional. Implements Table::find<Index>(...).
+  //
+  //     template <typename... Params>
+  //     Iterable range(kj::ArrayPtr<const Row> table, Params&&...) const;
+  //     // Optional. Implements Table::range<Index>(...).
+  //
+  //     Iterator begin() const;
+  //     Iterator end() const;
+  //     // Optional. Implements Table::ordered<Index>().
+  //   };
+
+public:
+  Table();
+  Table(Indexes&&... indexes);
+
+  void reserve(size_t size);
+  // Pre-allocates space for a table of the given size. Normally a Table grows by re-allocating
+  // its backing array whenever more space is needed. Reserving in advance avoids redundantly
+  // re-allocating as the table grows.
+
+  size_t size() const;
+  size_t capacity() const;
+
+  void clear();
+
+  Row* begin();
+  Row* end();
+  const Row* begin() const;
+  const Row* end() const;
+
+  Row& insert(Row&& row);
+  Row& insert(const Row& row);
+  // Inserts a new row. Throws an exception if this would violate the uniqueness constraints of any
+  // of the indexes.
+
+  template <typename Collection>
+  void insertAll(Collection&& collection);
+  template <typename Collection>
+  void insertAll(Collection& collection);
+  // Given an iterable collection of Rows, inserts all of them into this table. If the input is
+  // an rvalue, the rows will be moved rather than copied.
+
+  template <typename UpdateFunc>
+  Row& upsert(Row&& row, UpdateFunc&& update);
+  template <typename UpdateFunc>
+  Row& upsert(const Row& row, UpdateFunc&& update);
+  // Tries to insert a new row. However, if a duplicate already exists (according to some index),
+  // then update(Row& existingRow, Row&& newRow) is called to modify the existing row.
+
+  template <typename Index, typename... Params>
+  kj::Maybe<Row&> find(Params&&... params);
+  template <typename Index, typename... Params>
+  kj::Maybe<const Row&> find(Params&&... params) const;
+  // Using the given index, search for a matching row. What parameters are accepted depends on the
+  // index. Not all indexes support this method -- "multimap" indexes may support only range().
+
+  template <typename Index, typename... Params>
+  auto range(Params&&... params);
+  template <typename Index, typename... Params>
+  auto range(Params&&... params) const;
+  // Using the given index, look up a range of values, returning an iterable. What parameters are
+  // accepted depends on the index. Not all indexes support this method (in particular, unique
+  // indexes normally don't).
+
+  template <typename Index>
+  _::TableIterable<Row, Index&> ordered();
+  template <typename Index>
+  _::TableIterable<const Row, const Index&> ordered() const;
+  // Returns an iterable over the whole table ordered using the given index. Not all indexes
+  // support this method.
+
+  template <typename Index, typename... Params>
+  bool eraseMatch(Params&&... params);
+  // Erase the row that would be matched by `find<Index>(params)`. Returns true if there was a
+  // match.
+
+  template <typename Index, typename... Params>
+  size_t eraseRange(Params&&... params);
+  // Erase the row that would be matched by `range<Index>(params)`. Returns the number of
+  // elements erased.
+
+  void erase(Row& row);
+  // Erase the given row.
+  //
+  // WARNING: This invalidates all iterators, so you can't iterate over rows and erase them this
+  //   way. Use `eraseAll()` for that.
+
+  template <typename Predicate, typename = decltype(instance<Predicate>()(instance<Row&>()))>
+  size_t eraseAll(Predicate&& predicate);
+  // Erase all rows for which predicate(row) returns true. This scans over the entire table.
+
+  template <typename Collection, typename = decltype(instance<Collection>().begin()), bool = true>
+  size_t eraseAll(Collection&& collection);
+  // Erase all rows in the given iterable collection of rows. This carefully marks rows for
+  // deletion in a first pass then deletes them in a second.
+
+  template <size_t index = 0, typename... Params>
+  kj::Maybe<Row&> find(Params&&... params);
+  template <size_t index = 0, typename... Params>
+  kj::Maybe<const Row&> find(Params&&... params) const;
+  template <size_t index = 0, typename... Params>
+  auto range(Params&&... params);
+  template <size_t index = 0, typename... Params>
+  auto range(Params&&... params) const;
+  template <size_t index = 0>
+  _::TableIterable<Row, TypeOfIndex<index, Tuple<Indexes...>>&> ordered();
+  template <size_t index = 0>
+  _::TableIterable<const Row, const TypeOfIndex<index, Tuple<Indexes...>>&> ordered() const;
+  template <size_t index = 0, typename... Params>
+  bool eraseMatch(Params&&... params);
+  template <size_t index = 0, typename... Params>
+  size_t eraseRange(Params&&... params);
+  // Methods which take an index type as a template parameter can also take an index number. This
+  // is useful particularly when you have multiple indexes of the same type but different runtime
+  // properties. Additionally, you can omit the template parameter altogether to use the first
+  // index.
+
+  template <size_t index = 0>
+  void verify();
+  // Checks the integrity of indexes, throwing an exception if there are any problems. This is
+  // intended to be called within the unit test for an index.
+
+private:
+  Vector<Row> rows;
+  Tuple<Indexes...> indexes;
+
+  template <size_t index = 0, bool final = (index >= sizeof...(Indexes))>
+  class Impl;
+
+  void eraseImpl(size_t pos);
+  template <typename Collection>
+  size_t eraseAllImpl(Collection&& collection);
+};
+
+template <typename Callbacks>
+class HashIndex;
+// A Table index based on a hash table.
+//
+// This implementation:
+// * Is based on linear probing, not chaining. It is important to use a high-quality hash function.
+//   Use the KJ hashing library if possible.
+// * Is limited to tables of 2^30 rows or less, mainly to allow for tighter packing with 32-bit
+//   integers instead of 64-bit.
+// * Caches hash codes so that each table row need only be hashed once, and never checks equality
+//   unless hash codes have already been determined to be equal.
+//
+// The `Callbacks` type defines how to compute hash codes and equality. It should be defined like:
+//
+//   class Callbacks {
+//   public:
+//     bool matches(const Row&, const Row&) const;
+//     // Returns true if the two rows are matching, for the purpose of this index.
+//
+//     uint hashCode(const Row&) const;
+//     // Computes the hash code of the given row. Matching rows (as determined by match()) must
+//     // have the same hash code. Non-matching rows should have different hash codes, to the
+//     // maximum extent possible. Non-matching rows with the same hash code hurt performance.
+//
+//     bool matches(const Row&, ...) const;
+//     uint hashCode(...) const;
+//     // If you wish to support find(...) with parameters other than `const Row&`, you can do
+//     // so by implementing overloads of hashCode() and match() with those parameters.
+//   };
+//
+// If your `Callbacks` type has dynamic state, you may pass its constructor parameters as the
+// consturctor parameters to `HashIndex`.
+
+template <typename Callbacks>
+class TreeIndex;
+// A Table index based on a B-tree.
+//
+// This allows sorted iteration over rows.
+//
+// The `Callbacks` type defines how to compare rows. It should be defined like:
+//
+//   class Callbacks {
+//   public:
+//     bool isBefore(const Row&, const Row&) const;
+//     // Returns true if the row on te left comes before the row on the right.
+//
+//     bool matches(const Row&, const Row&) const;
+//     // Returns true if the rows "match".
+//     //
+//     // This could be computed by checking whether isBefore() returns false regardless of the
+//     // order of the parameters, but often equality is somewhat faster to check.
+//
+//     bool isBefore(const Row&, ...) const;
+//     bool matches(const Row&, ...) const;
+//     // If you wish to support find(...) with parameters other than `const Row&`, you can do
+//     // so by implementing overloads of isBefore() and isAfter() with those parameters.
+//   };
+
+// =======================================================================================
+// inline implementation details
+
+namespace _ {  // private
+
+KJ_NORETURN(void throwDuplicateTableRow());
+
+template <typename Dst, typename Src, typename = decltype(instance<Src>().size())>
+inline void tryReserveSize(Dst& dst, Src&& src) { dst.reserve(dst.size() + src.size()); }
+template <typename... Params>
+inline void tryReserveSize(Params&&...) {}
+// If `src` has a `.size()` method, call dst.reserve(dst.size() + src.size()).
+// Otherwise, do nothing.
+
+template <typename Inner, class Mapping>
+class MappedIterator: private Mapping {
+  // An iterator that wraps some other iterator and maps the values through a mapping function.
+  // The type `Mapping` must define a method `map()` which performs this mapping.
+  //
+  // TODO(cleanup): This seems generally useful. Should we put it somewhere resuable?
+
+public:
+  template <typename... Params>
+  MappedIterator(Inner inner, Params&&... params)
+      : Mapping(kj::fwd<Params>(params)...), inner(inner) {}
+
+  inline auto operator->() const { return &Mapping::map(*inner); }
+  inline decltype(auto) operator* () const { return Mapping::map(*inner); }
+  inline decltype(auto) operator[](size_t index) const { return Mapping::map(inner[index]); }
+  inline MappedIterator& operator++() { ++inner; return *this; }
+  inline MappedIterator  operator++(int) { return MappedIterator(inner++, *this); }
+  inline MappedIterator& operator--() { --inner; return *this; }
+  inline MappedIterator  operator--(int) { return MappedIterator(inner--, *this); }
+  inline MappedIterator& operator+=(ptrdiff_t amount) { inner += amount; return *this; }
+  inline MappedIterator& operator-=(ptrdiff_t amount) { inner -= amount; return *this; }
+  inline MappedIterator  operator+ (ptrdiff_t amount) const {
+    return MappedIterator(inner + amount, *this);
+  }
+  inline MappedIterator  operator- (ptrdiff_t amount) const {
+    return MappedIterator(inner - amount, *this);
+  }
+  inline ptrdiff_t operator- (const MappedIterator& other) const { return inner - other.inner; }
+
+  inline bool operator==(const MappedIterator& other) const { return inner == other.inner; }
+  inline bool operator!=(const MappedIterator& other) const { return inner != other.inner; }
+  inline bool operator<=(const MappedIterator& other) const { return inner <= other.inner; }
+  inline bool operator>=(const MappedIterator& other) const { return inner >= other.inner; }
+  inline bool operator< (const MappedIterator& other) const { return inner <  other.inner; }
+  inline bool operator> (const MappedIterator& other) const { return inner >  other.inner; }
+
+private:
+  Inner inner;
+};
+
+template <typename Inner, typename Mapping>
+class MappedIterable: private Mapping {
+  // An iterator that wraps some other iterator and maps the values through a mapping function.
+  // The type `Mapping` must define a method `map()` which performs this mapping.
+  //
+  // TODO(cleanup): This seems generally useful. Should we put it somewhere resuable?
+
+public:
+  template <typename... Params>
+  MappedIterable(Inner inner, Params&&... params)
+      : Mapping(kj::fwd<Params>(params)...), inner(inner) {}
+
+  typedef Decay<decltype(instance<Inner>().begin())> InnerIterator;
+  typedef MappedIterator<InnerIterator, Mapping> Iterator;
+  typedef Decay<decltype(instance<const Inner>().begin())> InnerConstIterator;
+  typedef MappedIterator<InnerConstIterator, Mapping> ConstIterator;
+
+  inline Iterator begin() { return { inner.begin(), (Mapping&)*this }; }
+  inline Iterator end() { return { inner.end(), (Mapping&)*this }; }
+  inline ConstIterator begin() const { return { inner.begin(), (const Mapping&)*this }; }
+  inline ConstIterator end() const { return { inner.end(), (const Mapping&)*this }; }
+
+private:
+  Inner inner;
+};
+
+template <typename Row>
+class TableMapping {
+public:
+  TableMapping(Row* table): table(table) {}
+  Row& map(size_t i) const { return table[i]; }
+
+private:
+  Row* table;
+};
+
+template <typename Row>
+class TableUnmapping {
+public:
+  TableUnmapping(Row* table): table(table) {}
+  size_t map(Row& row) const { return &row - table; }
+  size_t map(Row* row) const { return row - table; }
+
+private:
+  Row* table;
+};
+
+}  // namespace _ (private)
+
+template <typename Row, typename... Indexes>
+template <size_t index>
+class Table<Row, Indexes...>::Impl<index, false> {
+public:
+  static void reserve(Table<Row, Indexes...>& table, size_t size) {
+    get<index>(table.indexes).reserve(size);
+    Impl<index + 1>::reserve(table, size);
+  }
+
+  static void clear(Table<Row, Indexes...>& table) {
+    get<index>(table.indexes).clear();
+    Impl<index + 1>::clear(table);
+  }
+
+  static kj::Maybe<size_t> insert(Table<Row, Indexes...>& table, size_t pos) {
+    KJ_IF_MAYBE(existing, get<index>(table.indexes).insert(table.rows.asPtr(), pos)) {
+      return *existing;
+    }
+
+    bool success = false;
+    KJ_DEFER(if (!success) { get<index>(table.indexes).erase(table.rows.asPtr(), pos); });
+    auto result = Impl<index + 1>::insert(table, pos);
+    success = result == nullptr;
+    return result;
+  }
+
+  static void erase(Table<Row, Indexes...>& table, size_t pos) {
+    get<index>(table.indexes).erase(table.rows.asPtr(), pos);
+    Impl<index + 1>::erase(table, pos);
+  }
+
+  static void move(Table<Row, Indexes...>& table, size_t oldPos, size_t newPos) {
+    get<index>(table.indexes).move(table.rows.asPtr(), oldPos, newPos);
+    Impl<index + 1>::move(table, oldPos, newPos);
+  }
+};
+
+template <typename Row, typename... Indexes>
+template <size_t index>
+class Table<Row, Indexes...>::Impl<index, true> {
+public:
+  static void reserve(Table<Row, Indexes...>& table, size_t size) {}
+  static void clear(Table<Row, Indexes...>& table) {}
+  static kj::Maybe<size_t> insert(Table<Row, Indexes...>& table, size_t pos) { return nullptr; }
+  static void erase(Table<Row, Indexes...>& table, size_t pos) {}
+  static void move(Table<Row, Indexes...>& table, size_t oldPos, size_t newPos) {}
+};
+
+template <typename Row, typename... Indexes>
+Table<Row, Indexes...>::Table(): Table(Indexes()...) {}
+
+template <typename Row, typename... Indexes>
+Table<Row, Indexes...>::Table(Indexes&&... indexes)
+    : indexes(tuple(kj::fwd<Indexes&&>(indexes)...)) {}
+
+template <typename Row, typename... Indexes>
+void Table<Row, Indexes...>::reserve(size_t size) {
+  rows.reserve(size);
+  Impl<>::reserve(*this, size);
+}
+
+template <typename Row, typename... Indexes>
+size_t Table<Row, Indexes...>::size() const {
+  return rows.size();
+}
+template <typename Row, typename... Indexes>
+void Table<Row, Indexes...>::clear() {
+  Impl<>::clear(*this);
+  rows.clear();
+}
+template <typename Row, typename... Indexes>
+size_t Table<Row, Indexes...>::capacity() const {
+  return rows.capacity();
+}
+
+template <typename Row, typename... Indexes>
+Row* Table<Row, Indexes...>::begin() {
+  return rows.begin();
+}
+template <typename Row, typename... Indexes>
+Row* Table<Row, Indexes...>::end() {
+  return rows.end();
+}
+template <typename Row, typename... Indexes>
+const Row* Table<Row, Indexes...>::begin() const {
+  return rows.begin();
+}
+template <typename Row, typename... Indexes>
+const Row* Table<Row, Indexes...>::end() const {
+  return rows.end();
+}
+
+template <typename Row, typename... Indexes>
+Row& Table<Row, Indexes...>::insert(Row&& row) {
+  size_t pos = rows.size();
+  Row& rowRef = rows.add(kj::mv(row));
+  bool success = false;
+  KJ_DEFER({ if (!success) rows.removeLast(); });
+  KJ_IF_MAYBE(existing, Impl<>::insert(*this, pos)) {
+    _::throwDuplicateTableRow();
+  } else {
+    success = true;
+    return rowRef;
+  }
+}
+template <typename Row, typename... Indexes>
+Row& Table<Row, Indexes...>::insert(const Row& row) {
+  return insert(kj::cp(row));
+}
+
+template <typename Row, typename... Indexes>
+template <typename Collection>
+void Table<Row, Indexes...>::insertAll(Collection&& collection) {
+  _::tryReserveSize(*this, collection);
+  for (auto& row: collection) {
+    insert(kj::mv(row));
+  }
+}
+
+template <typename Row, typename... Indexes>
+template <typename Collection>
+void Table<Row, Indexes...>::insertAll(Collection& collection) {
+  _::tryReserveSize(*this, collection);
+  for (auto& row: collection) {
+    insert(row);
+  }
+}
+
+template <typename Row, typename... Indexes>
+template <typename UpdateFunc>
+Row& Table<Row, Indexes...>::upsert(Row&& row, UpdateFunc&& update) {
+  size_t pos = rows.size();
+  Row& rowRef = rows.add(kj::mv(row));
+  KJ_IF_MAYBE(existing, Impl<>::insert(*this, pos)) {
+    update(rows[*existing], kj::mv(rowRef));
+    rows.removeLast();
+    return rows[*existing];
+  } else {
+    return rowRef;
+  }
+}
+template <typename Row, typename... Indexes>
+template <typename UpdateFunc>
+Row& Table<Row, Indexes...>::upsert(const Row& row, UpdateFunc&& update) {
+  return upsert(kj::cp(row), kj::fwd<UpdateFunc>(update));
+}
+
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+kj::Maybe<Row&> Table<Row, Indexes...>::find(Params&&... params) {
+  return find<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+kj::Maybe<Row&> Table<Row, Indexes...>::find(Params&&... params) {
+  KJ_IF_MAYBE(pos, get<index>(indexes).find(rows.asPtr(), kj::fwd<Params>(params)...)) {
+    return rows[*pos];
+  } else {
+    return nullptr;
+  }
+}
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+kj::Maybe<const Row&> Table<Row, Indexes...>::find(Params&&... params) const {
+  return find<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+kj::Maybe<const Row&> Table<Row, Indexes...>::find(Params&&... params) const {
+  KJ_IF_MAYBE(pos, get<index>(indexes).find(rows.asPtr(), kj::fwd<Params>(params)...)) {
+    return rows[*pos];
+  } else {
+    return nullptr;
+  }
+}
+
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+auto Table<Row, Indexes...>::range(Params&&... params) {
+  return range<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+auto Table<Row, Indexes...>::range(Params&&... params) {
+  auto inner = get<index>(indexes).range(rows.asPtr(), kj::fwd<Params>(params)...);
+  return _::TableIterable<Row, decltype(inner)>(kj::mv(inner), rows.begin());
+}
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+auto Table<Row, Indexes...>::range(Params&&... params) const {
+  return range<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+auto Table<Row, Indexes...>::range(Params&&... params) const {
+  auto inner = get<index>(indexes).range(rows.asPtr(), kj::fwd<Params>(params)...);
+  return _::TableIterable<const Row, decltype(inner)>(kj::mv(inner), rows.begin());
+}
+
+template <typename Row, typename... Indexes>
+template <typename Index>
+_::TableIterable<Row, Index&> Table<Row, Indexes...>::ordered() {
+  return ordered<indexOfType<Index, Tuple<Indexes...>>()>();
+}
+template <typename Row, typename... Indexes>
+template <size_t index>
+_::TableIterable<Row, TypeOfIndex<index, Tuple<Indexes...>>&> Table<Row, Indexes...>::ordered() {
+  return { get<index>(indexes), rows.begin() };
+}
+template <typename Row, typename... Indexes>
+template <typename Index>
+_::TableIterable<const Row, const Index&> Table<Row, Indexes...>::ordered() const {
+  return ordered<indexOfType<Index, Tuple<Indexes...>>()>();
+}
+template <typename Row, typename... Indexes>
+template <size_t index>
+_::TableIterable<const Row, const TypeOfIndex<index, Tuple<Indexes...>>&>
+Table<Row, Indexes...>::ordered() const {
+  return { get<index>(indexes), rows.begin() };
+}
+
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+bool Table<Row, Indexes...>::eraseMatch(Params&&... params) {
+  return eraseMatch<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+bool Table<Row, Indexes...>::eraseMatch(Params&&... params) {
+  KJ_IF_MAYBE(pos, get<index>(indexes).find(rows.asPtr(), kj::fwd<Params>(params)...)) {
+    eraseImpl(*pos);
+    return true;
+  } else {
+    return false;
+  }
+}
+
+template <typename Row, typename... Indexes>
+template <typename Index, typename... Params>
+size_t Table<Row, Indexes...>::eraseRange(Params&&... params) {
+  return eraseRange<indexOfType<Index, Tuple<Indexes...>>()>(kj::fwd<Params>(params)...);
+}
+template <typename Row, typename... Indexes>
+template <size_t index, typename... Params>
+size_t Table<Row, Indexes...>::eraseRange(Params&&... params) {
+  return eraseAllImpl(get<index>(indexes).range(rows.asPtr(), kj::fwd<Params>(params)...));
+}
+
+template <typename Row, typename... Indexes>
+template <size_t index>
+void Table<Row, Indexes...>::verify() {
+  get<index>(indexes).verify(rows.asPtr());
+}
+
+template <typename Row, typename... Indexes>
+void Table<Row, Indexes...>::erase(Row& row) {
+  KJ_IREQUIRE(&row >= rows.begin() && &row < rows.end(), "row is not a member of this table");
+  eraseImpl(&row - rows.begin());
+}
+template <typename Row, typename... Indexes>
+void Table<Row, Indexes...>::eraseImpl(size_t pos) {
+  Impl<>::erase(*this, pos);
+  size_t back = rows.size() - 1;
+  if (pos != back) {
+    Impl<>::move(*this, back, pos);
+    rows[pos] = kj::mv(rows[back]);
+  }
+  rows.removeLast();
+}
+
+template <typename Row, typename... Indexes>
+template <typename Predicate, typename>
+size_t Table<Row, Indexes...>::eraseAll(Predicate&& predicate) {
+  size_t count = 0;
+  for (size_t i = 0; i < rows.size();) {
+    if (predicate(rows[i])) {
+      eraseImpl(i);
+      ++count;
+      // eraseImpl() replaces the erased row with the last row, so don't increment i here; repeat
+      // with the same i.
+    } else {
+      ++i;
+    }
+  }
+  return count;
+}
+
+template <typename Row, typename... Indexes>
+template <typename Collection, typename, bool>
+size_t Table<Row, Indexes...>::eraseAll(Collection&& collection) {
+  return eraseAllImpl(_::MappedIterable<Collection&, _::TableUnmapping<Row>>(
+      collection, rows.begin()));
+}
+
+template <typename Row, typename... Indexes>
+template <typename Collection>
+size_t Table<Row, Indexes...>::eraseAllImpl(Collection&& collection) {
+  // We need to transform the collection of row numbers into a sequence of erasures, accounting
+  // for the fact that each erasure re-positions the last row into its slot.
+  Vector<size_t> erased;
+  _::tryReserveSize(erased, collection);
+  for (size_t pos: collection) {
+    while (pos >= rows.size() - erased.size()) {
+      // Oops, the next item to be erased is already scheduled to be moved to a different location
+      // due to a previous erasure. Figure out where it will be at this point.
+      size_t erasureNumber = rows.size() - pos - 1;
+      pos = erased[erasureNumber];
+    }
+    erased.add(pos);
+  }
+
+  // Now we can execute the sequence of erasures.
+  for (size_t pos: erased) {
+    eraseImpl(pos);
+  }
+
+  return erased.size();
+}
+
+// -----------------------------------------------------------------------------
+// Hash table index
+
+namespace _ {  // private
+
+void logHashTableInconsistency();
+
+struct HashBucket {
+  uint hash;
+  uint value;
+
+  HashBucket() = default;
+  HashBucket(uint hash, uint pos)
+      : hash(hash), value(pos + 2) {}
+
+  inline bool isEmpty() const { return value == 0; }
+  inline bool isErased() const { return value == 1; }
+  inline bool isOccupied() const { return value >= 2; }
+  template <typename Row>
+  inline Row& getRow(ArrayPtr<Row> table) const { return table[getPos()]; }
+  template <typename Row>
+  inline const Row& getRow(ArrayPtr<const Row> table) const { return table[getPos()]; }
+  inline bool isPos(uint pos) const { return pos + 2 == value; }
+  inline uint getPos() const {
+    KJ_IASSERT(value >= 2);
+    return value - 2;
+  }
+  inline void setEmpty() { value = 0; }
+  inline void setErased() { value = 1; }
+  inline void setPos(uint pos) { value = pos + 2; }
+};
+
+inline size_t probeHash(const kj::Array<HashBucket>& buckets, size_t i) {
+  // TODO(perf): Is linear probing OK or should we do something fancier?
+  if (++i == buckets.size()) {
+    return 0;
+  } else {
+    return i;
+  }
+}
+
+kj::Array<HashBucket> rehash(kj::ArrayPtr<const HashBucket> oldBuckets, size_t targetSize);
+
+}  // namespace _ (private)
+
+template <typename Callbacks>
+class HashIndex {
+public:
+  HashIndex() = default;
+  template <typename... Params>
+  HashIndex(Params&&... params): cb(kj::fwd<Params>(params)...) {}
+
+  void reserve(size_t size) {
+    if (buckets.size() < size * 2) {
+      rehash(size);
+    }
+  }
+
+  void clear() {
+    erasedCount = 0;
+    memset(buckets.begin(), 0, buckets.asBytes().size());
+  }
+
+  template <typename Row>
+  kj::Maybe<size_t> insert(kj::ArrayPtr<Row> table, size_t pos) {
+    if (buckets.size() * 2 < (table.size() + erasedCount) * 3) {
+      // Load factor is more than 2/3, let's rehash.
+      rehash(kj::max(buckets.size() * 2, table.size() * 2));
+    }
+
+    uint hashCode = cb.hashCode(table[pos]);
+    Maybe<_::HashBucket&> erasedSlot;
+    for (uint i = hashCode % buckets.size();; i = _::probeHash(buckets, i)) {
+      auto& bucket = buckets[i];
+      if (bucket.isEmpty()) {
+        // no duplicates found
+        KJ_IF_MAYBE(s, erasedSlot) {
+          --erasedCount;
+          *s = { hashCode, uint(pos) };
+        } else {
+          bucket = { hashCode, uint(pos) };
+        }
+        return nullptr;
+      } else if (bucket.isErased()) {
+        // We can fill in the erased slot. However, we have to keep searching to make sure there
+        // are no duplicates before we do that.
+        if (erasedSlot == nullptr) {
+          erasedSlot = bucket;
+        }
+      } else if (bucket.hash == hashCode &&
+                 cb.matches(bucket.getRow(table), table[pos])) {
+        // duplicate row
+        return size_t(bucket.getPos());
+      }
+    }
+  }
+
+  template <typename Row>
+  void erase(kj::ArrayPtr<Row> table, size_t pos) {
+    uint hashCode = cb.hashCode(table[pos]);
+    for (uint i = hashCode % buckets.size();; i = _::probeHash(buckets, i)) {
+      auto& bucket = buckets[i];
+      if (bucket.isPos(pos)) {
+        // found it
+        ++erasedCount;
+        bucket.setErased();
+        return;
+      } else if (bucket.isEmpty()) {
+        // can't find the bucket, something is very wrong
+        _::logHashTableInconsistency();
+        return;
+      }
+    }
+  }
+
+  template <typename Row>
+  void move(kj::ArrayPtr<Row> table, size_t oldPos, size_t newPos) {
+    uint hashCode = cb.hashCode(table[oldPos]);
+    for (uint i = hashCode % buckets.size();; i = _::probeHash(buckets, i)) {
+      auto& bucket = buckets[i];
+      if (bucket.isPos(oldPos)) {
+        // found it
+        bucket.setPos(newPos);
+        return;
+      } else if (bucket.isEmpty()) {
+        // can't find the bucket, something is very wrong
+        _::logHashTableInconsistency();
+        return;
+      }
+    }
+  }
+
+  template <typename Row, typename... Params>
+  Maybe<size_t> find(kj::ArrayPtr<Row> table, Params&&... params) const {
+    if (buckets.size() == 0) return nullptr;
+
+    uint hashCode = cb.hashCode(params...);
+    for (uint i = hashCode % buckets.size();; i = _::probeHash(buckets, i)) {
+      auto& bucket = buckets[i];
+      if (bucket.isEmpty()) {
+        // not found.
+        return nullptr;
+      } else if (bucket.isErased()) {
+        // skip, keep searching
+      } else if (bucket.hash == hashCode &&
+                 cb.matches(bucket.getRow(table), params...)) {
+        // found
+        return size_t(bucket.getPos());
+      }
+    }
+  }
+
+  // No begin() nor end() because hash tables are not usefully ordered.
+
+private:
+  Callbacks cb;
+  size_t erasedCount = 0;
+  Array<_::HashBucket> buckets;
+
+  void rehash(size_t targetSize) {
+    buckets = _::rehash(buckets, targetSize);
+  }
+};
+
+// -----------------------------------------------------------------------------
+// BTree index
+
+namespace _ {  // private
+
+KJ_ALWAYS_INLINE(void compilerBarrier()) {
+  // Make sure that reads occurring before this call cannot be re-ordered to happen after
+  // writes that occur after this call. We need this in a couple places below to prevent C++
+  // strict aliasing rules from breaking things.
+#if _MSC_VER
+  _ReadWriteBarrier()
+#else
+  __asm__ __volatile__("": : :"memory");
+#endif
+}
+
+template <typename T>
+inline void acopy(T* to, T* from, size_t size) { memcpy(to, from, size * sizeof(T)); }
+template <typename T>
+inline void amove(T* to, T* from, size_t size) { memmove(to, from, size * sizeof(T)); }
+template <typename T>
+inline void azero(T* ptr, size_t size) { memset(ptr, 0, size * sizeof(T)); }
+// memcpy/memmove/memset variants that count size in elements, not bytes.
+//
+// TODO(cleanup): These are generally useful, put them somewhere.
+
+class BTreeImpl {
+public:
+  class Iterator;
+  class MaybeUint;
+  struct NodeUnion;
+  struct Leaf;
+  struct Parent;
+  struct Freelisted;
+
+  class SearchKey {
+    // Passed to methods that need to search the tree. This class allows most of the B-tree
+    // implementation to be kept out of templates, avoiding code bloat, at the cost of some
+    // performance trade-off. In order to lessen the performance cost of virtual calls, we design
+    // this interface so that it only needs to be called once per tree node, rather than once per
+    // comparison.
+
+  public:
+    virtual uint search(const Parent& parent) const = 0;
+    virtual uint search(const Leaf& leaf) const = 0;
+    // Binary search for the first key/row in the parent/leaf that is equal to or comes after the
+    // search key.
+
+    virtual bool isAfter(uint rowIndex) const = 0;
+    // Returns true if the key comes after the value in the given row.
+  };
+
+  BTreeImpl();
+  ~BTreeImpl() noexcept(false);
+
+  void logInconsitency() const;
+
+  void reserve(size_t size);
+
+  void clear();
+
+  Iterator begin() const;
+  Iterator end() const;
+
+  Iterator search(const SearchKey& searchKey) const;
+  // Find the "first" row number (in sorted order) for which predicate(rowNumber) returns false.
+
+  Iterator insert(const SearchKey& searchKey);
+  // Like search() but ensures that there is room in the leaf node to insert a new row.
+
+  void erase(uint row, const SearchKey& searchKey);
+  // Erase the given row number from the tree. predicate() returns false for the given row and all
+  // rows after it.
+
+  void renumber(uint oldRow, uint newRow, const SearchKey& searchKey);
+  // Renumber the given row from oldRow to newRow. predicate() returns false for oldRow and all
+  // rows after it. (It will not be called on newRow.)
+
+  void verify(size_t size, FunctionParam<bool(uint, uint)>);
+
+private:
+  NodeUnion* tree;        // allocated with aligned_alloc aligned to cache lines
+  uint treeCapacity;
+  uint height;            // height of *parent* tree -- does not include the leaf level
+  uint freelistHead;
+  uint freelistSize;
+  uint beginLeaf;
+  uint endLeaf;
+  void growTree(uint minCapacity = 0);
+
+  template <typename T>
+  struct AllocResult;
+
+  template <typename T>
+  inline AllocResult<T> alloc();
+  inline void free(uint pos);
+
+  inline uint split(Parent& src, uint srcPos, Parent& dst, uint dstPos);
+  inline uint split(Leaf& dst, uint dstPos, Leaf& src, uint srcPos);
+  inline void merge(Parent& dst, uint dstPos, uint pivot, Parent& src);
+  inline void merge(Leaf& dst, uint dstPos, uint pivot, Leaf& src);
+  inline void move(Parent& dst, uint dstPos, Parent& src);
+  inline void move(Leaf& dst, uint dstPos, Leaf& src);
+  inline void rotateLeft(
+      Parent& left, Parent& right, Parent& parent, uint indexInParent, MaybeUint*& fixup);
+  inline void rotateLeft(
+      Leaf& left, Leaf& right, Parent& parent, uint indexInParent, MaybeUint*& fixup);
+  inline void rotateRight(Parent& left, Parent& right, Parent& parent, uint indexInParent);
+  inline void rotateRight(Leaf& left, Leaf& right, Parent& parent, uint indexInParent);
+
+  template <typename Node>
+  inline Node& insertHelper(const SearchKey& searchKey,
+      Node& node, Parent* parent, uint indexInParent, uint pos);
+
+  template <typename Node>
+  inline Node& eraseHelper(
+      Node& node, Parent* parent, uint indexInParent, uint pos, MaybeUint*& fixup);
+
+  size_t verifyNode(size_t size, FunctionParam<bool(uint, uint)>&,
+                    uint pos, uint height, MaybeUint maxRow);
+
+  static const NodeUnion EMPTY_NODE;
+};
+
+class BTreeImpl::MaybeUint {
+  // A nullable uint, using the value zero to mean null and shifting all other values up by 1.
+public:
+  MaybeUint() = default;
+  inline MaybeUint(uint i): i(i - 1) {}
+  inline MaybeUint(decltype(nullptr)): i(0) {}
+
+  inline bool operator==(decltype(nullptr)) const { return i == 0; }
+  inline bool operator==(uint j) const { return i == j + 1; }
+  inline bool operator==(const MaybeUint& other) const { return i == other.i; }
+  inline bool operator!=(decltype(nullptr)) const { return i != 0; }
+  inline bool operator!=(uint j) const { return i != j + 1; }
+  inline bool operator!=(const MaybeUint& other) const { return i != other.i; }
+
+  inline MaybeUint& operator=(decltype(nullptr)) { i = 0; return *this; }
+  inline MaybeUint& operator=(uint j) { i = j + 1; return *this; }
+
+  inline uint operator*() const { KJ_IREQUIRE(i != 0); return i - 1; }
+
+  template <typename Func>
+  inline bool check(Func& func) const { return i != 0 && func(i - 1); }
+  // Equivalent to *this != nullptr && func(**this)
+
+private:
+  uint i;
+};
+
+struct BTreeImpl::Leaf {
+  uint next;
+  uint prev;
+  // Pointers to next and previous nodes at the same level, used for fast iteration.
+
+  MaybeUint rows[14];
+  // Pointers to table rows, offset by 1 so that 0 is an empty value.
+
+  inline bool isFull() const;
+  inline bool isMostlyFull() const;
+  inline bool isHalfFull() const;
+
+  inline void insert(uint i, uint newRow) {
+    KJ_IREQUIRE(rows[kj::size(&Leaf::rows) - 1] == nullptr);  // check not full
+
+    amove(rows + i + 1, rows + i, kj::size(&Leaf::rows) - (i + 1));
+    rows[i] = newRow;
+  }
+
+  inline void erase(uint i) {
+    KJ_IREQUIRE(rows[0] != nullptr);  // check not empty
+
+    amove(rows + i, rows + i + 1, kj::size(&Leaf::rows) - (i + 1));
+    rows[kj::size(&Leaf::rows) - 1] = nullptr;
+  }
+
+  inline uint size() const {
+    static_assert(kj::size(&Leaf::rows) == 14, "logic here needs updating");
+
+    // Binary search for first empty element in `rows`, or return 14 if no empty elements. We do
+    // this in a branch-free manner. Since there are 15 possible results (0 through 14, inclusive),
+    // this isn't a perfectly balanced binary search. We carefully choose the split points so that
+    // there's no way we'll try to dereference row[14] or later (which would be a buffer overflow).
+    uint i = (rows[6] != nullptr) * 7;
+    i += (rows[i + 3] != nullptr) * 4;
+    i += (rows[i + 1] != nullptr) * 2;
+    i += (rows[i    ] != nullptr);
+    return i;
+  }
+
+  template <typename Func>
+  inline uint binarySearch(Func& predicate) const {
+    // Binary search to find first row for which predicate(row) is false.
+
+    static_assert(kj::size(&Leaf::rows) == 14, "logic here needs updating");
+
+    // See comments in size().
+    uint i = (rows[6].check(predicate)) * 7;
+    i += (rows[i + 3].check(predicate)) * 4;
+    i += (rows[i + 1].check(predicate)) * 2;
+    if (i != 6) {  // don't redundantly check row 6
+      i += (rows[i    ].check(predicate));
+    }
+    return i;
+  }
+};
+
+struct BTreeImpl::Parent {
+  uint unused;
+  // Not used. May be arbitrarily non-zero due to overlap with Freelisted::nextOffset.
+
+  MaybeUint keys[7];
+  // Pointers to table rows, offset by 1 so that 0 is an empty value.
+
+  uint children[kj::size(&Parent::keys) + 1];
+  // Pointers to children. Not offset because the root is always at position 0, and a pointer
+  // to the root would be nonsensical.
+
+  inline bool isFull() const;
+  inline bool isMostlyFull() const;
+  inline bool isHalfFull() const;
+  inline void initRoot(uint key, uint leftChild, uint rightChild);
+  inline void insertAfter(uint i, uint splitKey, uint child);
+  inline void eraseAfter(uint i);
+
+  inline uint keyCount() const {
+    static_assert(kj::size(&Parent::keys) == 7, "logic here needs updating");
+
+    // Binary search for first empty element in `keys`, or return 7 if no empty elements. We do
+    // this in a branch-free manner. Since there are 8 possible results (0 through 7, inclusive),
+    // this is a perfectly balanced binary search.
+    uint i = (keys[3] != nullptr) * 4;
+    i += (keys[i + 1] != nullptr) * 2;
+    i += (keys[i    ] != nullptr);
+    return i;
+  }
+
+  template <typename Func>
+  inline uint binarySearch(Func& predicate) const {
+    // Binary search to find first key for which predicate(key) is false.
+
+    static_assert(kj::size(&Parent::keys) == 7, "logic here needs updating");
+
+    // See comments in size().
+    uint i = (keys[3].check(predicate)) * 4;
+    i += (keys[i + 1].check(predicate)) * 2;
+    i += (keys[i    ].check(predicate));
+    return i;
+  }
+};
+
+struct BTreeImpl::Freelisted {
+  int nextOffset;
+  // The next node in the freelist is at: this + 1 + nextOffset
+  //
+  // Hence, newly-allocated space can initialize this to zero.
+
+  uint zero[15];
+  // Freelisted entries are always zero'd.
+};
+
+struct BTreeImpl::NodeUnion {
+  union {
+    Freelisted freelist;
+    // If this node is in the freelist.
+
+    Leaf leaf;
+    // If this node is a leaf.
+
+    Parent parent;
+    // If this node is not a leaf.
+  };
+
+  inline operator Leaf&() { return leaf; }
+  inline operator Parent&() { return parent; }
+  inline operator const Leaf&() const { return leaf; }
+  inline operator const Parent&() const { return parent; }
+};
+
+static_assert(sizeof(BTreeImpl::Parent) == 64,
+    "BTreeImpl::Parent should be optimized to fit a cache line");
+static_assert(sizeof(BTreeImpl::Leaf) == 64,
+    "BTreeImpl::Leaf should be optimized to fit a cache line");
+static_assert(sizeof(BTreeImpl::Freelisted) == 64,
+    "BTreeImpl::Freelisted should be optimized to fit a cache line");
+static_assert(sizeof(BTreeImpl::NodeUnion) == 64,
+    "BTreeImpl::NodeUnion should be optimized to fit a cache line");
+
+bool BTreeImpl::Leaf::isFull() const {
+  return rows[kj::size(&Leaf::rows) - 1] != nullptr;
+}
+bool BTreeImpl::Leaf::isMostlyFull() const {
+  return rows[kj::size(&Leaf::rows) / 2] != nullptr;
+}
+bool BTreeImpl::Leaf::isHalfFull() const {
+  KJ_IASSERT(rows[kj::size(&Leaf::rows) / 2 - 1] != nullptr);
+  return rows[kj::size(&Leaf::rows) / 2] == nullptr;
+}
+
+bool BTreeImpl::Parent::isFull() const {
+  return keys[kj::size(&Parent::keys) - 1] != nullptr;
+}
+bool BTreeImpl::Parent::isMostlyFull() const {
+  return keys[kj::size(&Parent::keys) / 2] != nullptr;
+}
+bool BTreeImpl::Parent::isHalfFull() const {
+  KJ_IASSERT(keys[kj::size(&Parent::keys) / 2 - 1] != nullptr);
+  return keys[kj::size(&Parent::keys) / 2] == nullptr;
+}
+
+class BTreeImpl::Iterator {
+public:
+  Iterator(const NodeUnion* tree, const Leaf* leaf, uint row)
+      : tree(tree), leaf(leaf), row(row) {}
+
+  size_t operator*() const {
+    KJ_IREQUIRE(row < kj::size(&Leaf::rows) && leaf->rows[row] != nullptr,
+        "tried to dereference end() iterator");
+    return *leaf->rows[row];
+  }
+
+  inline Iterator& operator++() {
+    KJ_IREQUIRE(leaf->rows[row] != nullptr, "B-tree iterator overflow");
+    ++row;
+    if (row >= kj::size(&Leaf::rows) || leaf->rows[row] == nullptr) {
+      if (leaf->next == 0) {
+        // at end; stay on current leaf
+      } else {
+        leaf = &tree[leaf->next].leaf;
+        row = 0;
+      }
+    }
+    return *this;
+  }
+  inline Iterator operator++(int) {
+    Iterator other = *this;
+    ++*this;
+    return other;
+  }
+
+  inline Iterator& operator--() {
+    if (row == 0) {
+      KJ_IREQUIRE(leaf->prev != 0, "B-tree iterator underflow");
+      leaf = &tree[leaf->prev].leaf;
+      row = leaf->size() - 1;
+    } else {
+      --row;
+    }
+    return *this;
+  }
+  inline Iterator operator--(int) {
+    Iterator other = *this;
+    --*this;
+    return other;
+  }
+
+  inline bool operator==(const Iterator& other) const {
+    return leaf == other.leaf && row == other.row;
+  }
+  inline bool operator!=(const Iterator& other) const {
+    return leaf != other.leaf || row != other.row;
+  }
+
+  bool isEnd() {
+    return row == kj::size(&Leaf::rows) || leaf->rows[row] == nullptr;
+  }
+
+  void insert(BTreeImpl& impl, uint newRow) {
+    KJ_IASSERT(impl.tree == tree);
+    const_cast<Leaf*>(leaf)->insert(row, newRow);
+  }
+
+  void erase(BTreeImpl& impl) {
+    KJ_IASSERT(impl.tree == tree);
+    const_cast<Leaf*>(leaf)->erase(row);
+  }
+
+  void replace(BTreeImpl& impl, uint newRow) {
+    KJ_IASSERT(impl.tree == tree);
+    const_cast<Leaf*>(leaf)->rows[row] = newRow;
+  }
+
+private:
+  const NodeUnion* tree;
+  const Leaf* leaf;
+  uint row;
+};
+
+template <typename Iterator>
+class IterRange {
+public:
+  inline IterRange(Iterator b, Iterator e): b(b), e(e) {}
+
+  inline Iterator begin() const { return b; }
+  inline Iterator end() const { return e; }
+private:
+  Iterator b;
+  Iterator e;
+};
+
+template <typename Iterator>
+inline IterRange<Decay<Iterator>> iterRange(Iterator b, Iterator e) {
+  return { b, e };
+}
+
+inline BTreeImpl::Iterator BTreeImpl::begin() const {
+  return { tree, &tree[beginLeaf].leaf, 0 };
+}
+inline BTreeImpl::Iterator BTreeImpl::end() const {
+  auto& leaf = tree[endLeaf].leaf;
+  return { tree, &leaf, leaf.size() };
+}
+
+}  // namespace _ (private)
+
+template <typename Callbacks>
+class TreeIndex {
+public:
+  TreeIndex() = default;
+  template <typename... Params>
+  TreeIndex(Params&&... params): cb(kj::fwd<Params>(params)...) {}
+
+  template <typename Row>
+  void verify(kj::ArrayPtr<Row> table) { // KJ_DBG
+    impl.verify(table.size(), [&](uint i, uint j) {
+      return cb.isBefore(table[i], table[j]);
+    });
+  }
+
+  inline void reserve(size_t size) { impl.reserve(size); }
+  inline void clear() { impl.clear(); }
+  inline auto begin() const { return impl.begin(); }
+  inline auto end() const { return impl.end(); }
+
+  template <typename Row>
+  kj::Maybe<size_t> insert(kj::ArrayPtr<Row> table, size_t pos) {
+    auto& newRow = table[pos];
+    auto iter = impl.insert(searchKey(table, newRow));
+
+    if (!iter.isEnd() && cb.matches(table[*iter], newRow)) {
+      return *iter;
+    } else {
+      iter.insert(impl, pos);
+      return nullptr;
+    }
+  }
+
+  template <typename Row>
+  void erase(kj::ArrayPtr<Row> table, size_t pos) {
+    auto& row = table[pos];
+    impl.erase(pos, searchKey(table, row));
+  }
+
+  template <typename Row>
+  void move(kj::ArrayPtr<Row> table, size_t oldPos, size_t newPos) {
+    auto& row = table[oldPos];
+    impl.renumber(oldPos, newPos, searchKey(table, row));
+  }
+
+  template <typename Row, typename... Params>
+  Maybe<size_t> find(kj::ArrayPtr<Row> table, Params&&... params) const {
+    auto iter = impl.search(searchKey(table, params...));
+
+    if (!iter.isEnd() && cb.matches(table[*iter], params...)) {
+      return size_t(*iter);
+    } else {
+      return nullptr;
+    }
+  }
+
+  template <typename Row, typename Begin, typename End>
+  _::IterRange<_::BTreeImpl::Iterator> range(
+      kj::ArrayPtr<Row> table, Begin&& begin, End&& end) const {
+    return {
+      impl.search(searchKey(table, begin)),
+      impl.search(searchKey(table, end  ))
+    };
+  }
+
+private:
+  Callbacks cb;
+  _::BTreeImpl impl;
+
+  template <typename Predicate>
+  class SearchKeyImpl: public _::BTreeImpl::SearchKey {
+  public:
+    SearchKeyImpl(Predicate&& predicate)
+        : predicate(kj::mv(predicate)) {}
+
+    uint search(const _::BTreeImpl::Parent& parent) const override {
+      return parent.binarySearch(predicate);
+    }
+    uint search(const _::BTreeImpl::Leaf& leaf) const override {
+      return leaf.binarySearch(predicate);
+    }
+    bool isAfter(uint rowIndex) const override {
+      return predicate(rowIndex);
+    }
+
+  private:
+    Predicate predicate;
+  };
+
+  template <typename Row, typename... Params>
+  inline auto searchKey(kj::ArrayPtr<Row>& table, Params&... params) const {
+    auto predicate = [&](uint i) { return cb.isBefore(table[i], params...); };
+    return SearchKeyImpl<decltype(predicate)>(kj::mv(predicate));
+  }
+};
+
+} // namespace kj