layout.c++ 143 KB
Newer Older
1
// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
Kenton Varda's avatar
Kenton Varda committed
2
// Licensed under the MIT License:
3
//
Kenton Varda's avatar
Kenton Varda committed
4 5 6 7 8 9
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
10
//
Kenton Varda's avatar
Kenton Varda committed
11 12
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
13
//
Kenton Varda's avatar
Kenton Varda committed
14 15 16 17 18 19 20
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
21

Kenton Varda's avatar
Kenton Varda committed
22
#define CAPNP_PRIVATE
23
#include "layout.h"
Kenton Varda's avatar
Kenton Varda committed
24
#include <kj/debug.h>
25
#include "arena.h"
26
#include <string.h>
27
#include <stdlib.h>
28

29 30 31 32
#if !CAPNP_LITE
#include "capability.h"
#endif  // !CAPNP_LITE

33
namespace capnp {
34
namespace _ {  // private
Kenton Varda's avatar
Kenton Varda committed
35

36
#if !CAPNP_LITE
37 38 39 40 41
static BrokenCapFactory* brokenCapFactory = nullptr;
// Horrible hack:  We need to be able to construct broken caps without any capability context,
// but we can't have a link-time dependency on libcapnp-rpc.

void setGlobalBrokenCapFactoryForLayoutCpp(BrokenCapFactory& factory) {
42 43
  // Called from capability.c++ when the capability API is used, to make sure that layout.c++
  // is ready for it.  May be called multiple times but always with the same value.
44
#if __GNUC__
45
  __atomic_store_n(&brokenCapFactory, &factory, __ATOMIC_RELAXED);
46 47 48 49 50
#elif _MSC_VER
  *static_cast<BrokenCapFactory* volatile*>(&brokenCapFactory) = &factory;
#else
#error "Platform not supported"
#endif
51
}
Kenton Varda's avatar
Kenton Varda committed
52 53 54 55 56 57 58 59

}  // namespace _ (private)

const uint ClientHook::NULL_CAPABILITY_BRAND = 0;
// Defined here rather than capability.c++ so that we can safely call isNull() in this file.

namespace _ {  // private

60
#endif  // !CAPNP_LITE
61

62
#if CAPNP_DEBUG_TYPES
63
#define G(n) bounded<n>()
64 65 66
#else
#define G(n) n
#endif
67

68 69
// =======================================================================================

70 71
struct WirePointer {
  // A pointer, in exactly the format in which it appears on the wire.
72 73

  // Copying and moving is not allowed because the offset would become wrong.
74 75 76 77
  WirePointer(const WirePointer& other) = delete;
  WirePointer(WirePointer&& other) = delete;
  WirePointer& operator=(const WirePointer& other) = delete;
  WirePointer& operator=(WirePointer&& other) = delete;
78

79
  // -----------------------------------------------------------------
80
  // Common part of all pointers:  kind + offset
81 82 83
  //
  // Actually this is not terribly common.  The "offset" could actually be different things
  // depending on the context:
84 85
  // - For a regular (e.g. struct/list) pointer, a signed word offset from the word immediately
  //   following the pointer pointer.  (The off-by-one means the offset is more often zero, saving
86
  //   bytes on the wire when packed.)
87
  // - For an inline composite list tag (not really a pointer, but structured similarly), an
88
  //   element count.
89
  // - For a FAR pointer, an unsigned offset into the target segment.
90
  // - For a FAR landing pad, zero indicates that the target value immediately follows the pad while
91
  //   1 indicates that the pad is followed by another FAR pointer that actually points at the
92 93 94
  //   value.

  enum Kind {
95
    STRUCT = 0,
96 97
    // Reference points at / describes a struct.

98
    LIST = 1,
99 100 101 102 103 104
    // Reference points at / describes a list.

    FAR = 2,
    // Reference is a "far pointer", which points at data located in a different segment.  The
    // eventual target is one of the other kinds.

105 106 107
    OTHER = 3
    // Reference has type "other".  If the next 30 bits are all zero (i.e. the lower 32 bits contain
    // only the kind OTHER) then the pointer is a capability.  All other values are reserved.
108 109
  };

110 111
  WireValue<uint32_t> offsetAndKind;

112
  KJ_ALWAYS_INLINE(Kind kind() const) {
113 114
    return static_cast<Kind>(offsetAndKind.get() & 3);
  }
115 116 117 118 119 120
  KJ_ALWAYS_INLINE(bool isPositional() const) {
    return (offsetAndKind.get() & 2) == 0;  // match STRUCT and LIST but not FAR or OTHER
  }
  KJ_ALWAYS_INLINE(bool isCapability() const) {
    return offsetAndKind.get() == OTHER;
  }
121

122
  KJ_ALWAYS_INLINE(word* target()) {
123
    return reinterpret_cast<word*>(this) + 1 + (static_cast<int32_t>(offsetAndKind.get()) >> 2);
124
  }
125
  KJ_ALWAYS_INLINE(const word* target() const) {
126 127
    return reinterpret_cast<const word*>(this) + 1 +
        (static_cast<int32_t>(offsetAndKind.get()) >> 2);
128
  }
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
  KJ_ALWAYS_INLINE(void setKindAndTarget(Kind kind, word* target, SegmentBuilder* segment)) {
    // Check that the target is really in the same segment, otherwise subtracting pointers is
    // undefined behavior.  As it turns out, it's undefined behavior that actually produces
    // unexpected results in a real-world situation that actually happened:  At one time,
    // OrphanBuilder's "tag" (a WirePointer) was allowed to be initialized as if it lived in
    // a particular segment when in fact it does not.  On 32-bit systems, where words might
    // only be 32-bit aligned, it's possible that the difference between `this` and `target` is
    // not a whole number of words.  But clang optimizes:
    //     (target - (word*)this - 1) << 2
    // to:
    //     (((ptrdiff_t)target - (ptrdiff_t)this - 8) >> 1)
    // So now when the pointers are not aligned the same, we can end up corrupting the bottom
    // two bits, where `kind` is stored.  For example, this turns a struct into a far pointer.
    // Ouch!
    KJ_DREQUIRE(segment->containsInterval(
        reinterpret_cast<word*>(this), reinterpret_cast<word*>(this + 1)));
    KJ_DREQUIRE(segment->containsInterval(target, target));
146
    offsetAndKind.set(((target - reinterpret_cast<word*>(this) - 1) << 2) | kind);
147
  }
148
  KJ_ALWAYS_INLINE(void setKindWithZeroOffset(Kind kind)) {
149 150
    offsetAndKind.set(kind);
  }
151 152 153 154 155 156 157 158 159 160
  KJ_ALWAYS_INLINE(void setKindAndTargetForEmptyStruct()) {
    // This pointer points at an empty struct.  Assuming the WirePointer itself is in-bounds, we
    // can set the target to point either at the WirePointer itself or immediately after it.  The
    // latter would cause the WirePointer to be "null" (since for an empty struct the upper 32
    // bits are going to be zero).  So we set an offset of -1, as if the struct were allocated
    // immediately before this pointer, to distinguish it from null.
    offsetAndKind.set(0xfffffffc);
  }
  KJ_ALWAYS_INLINE(void setKindForOrphan(Kind kind)) {
    // OrphanBuilder contains a WirePointer, but since it isn't located in a segment, it should
161 162 163 164
    // not have a valid offset (unless it is a FAR or OTHER pointer).  We set its offset to -1
    // because setting it to zero would mean a pointer to an empty struct would appear to be a null
    // pointer.
    KJ_DREQUIRE(isPositional());
165 166
    offsetAndKind.set(kind | 0xfffffffc);
  }
167

168
  KJ_ALWAYS_INLINE(ListElementCount inlineCompositeListElementCount() const) {
169
    return ((bounded(offsetAndKind.get()) >> G(2))
170
            & G(kj::maxValueForBits<LIST_ELEMENT_COUNT_BITS>())) * ELEMENTS;
171
  }
172
  KJ_ALWAYS_INLINE(void setKindAndInlineCompositeListElementCount(
173
      Kind kind, ListElementCount elementCount)) {
174
    offsetAndKind.set(unboundAs<uint32_t>((elementCount / ELEMENTS) << G(2)) | kind);
175 176
  }

177
  KJ_ALWAYS_INLINE(SegmentWordCount farPositionInSegment() const) {
178
    KJ_DREQUIRE(kind() == FAR,
179
        "positionInSegment() should only be called on FAR pointers.");
180
    return (bounded(offsetAndKind.get()) >> G(3)) * WORDS;
181
  }
182
  KJ_ALWAYS_INLINE(bool isDoubleFar() const) {
183
    KJ_DREQUIRE(kind() == FAR,
184
        "isDoubleFar() should only be called on FAR pointers.");
185
    return unbound((bounded(offsetAndKind.get()) >> G(2)) & G(1));
186
  }
187
  KJ_ALWAYS_INLINE(void setFar(bool isDoubleFar, WordCountN<29> pos)) {
188
    offsetAndKind.set(unboundAs<uint32_t>((pos / WORDS) << G(3)) |
189
                      (static_cast<uint32_t>(isDoubleFar) << 2) |
190
                      static_cast<uint32_t>(Kind::FAR));
191
  }
192 193 194 195
  KJ_ALWAYS_INLINE(void setCap(uint index)) {
    offsetAndKind.set(static_cast<uint32_t>(Kind::OTHER));
    capRef.index.set(index);
  }
196 197

  // -----------------------------------------------------------------
198
  // Part of pointer that depends on the kind.
199

200 201 202 203 204 205 206
  // Note:  Originally StructRef, ListRef, and FarRef were unnamed types, but this somehow
  //   tickled a bug in GCC:
  //     http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58192
  struct StructRef {
    WireValue<WordCount16> dataSize;
    WireValue<WirePointerCount16> ptrCount;

207
    inline WordCountN<17> wordSize() const {
208
      return upgradeBound<uint32_t>(dataSize.get()) + ptrCount.get() * WORDS_PER_POINTER;
209 210
    }

211
    KJ_ALWAYS_INLINE(void set(WordCount16 ds, WirePointerCount16 rc)) {
212 213 214 215 216 217 218 219 220 221 222 223
      dataSize.set(ds);
      ptrCount.set(rc);
    }
    KJ_ALWAYS_INLINE(void set(StructSize size)) {
      dataSize.set(size.data);
      ptrCount.set(size.pointers);
    }
  };

  struct ListRef {
    WireValue<uint32_t> elementSizeAndCount;

224 225
    KJ_ALWAYS_INLINE(ElementSize elementSize() const) {
      return static_cast<ElementSize>(elementSizeAndCount.get() & 7);
226
    }
227
    KJ_ALWAYS_INLINE(ElementCountN<29> elementCount() const) {
228
      return (bounded(elementSizeAndCount.get()) >> G(3)) * ELEMENTS;
229
    }
230 231
    KJ_ALWAYS_INLINE(WordCountN<29> inlineCompositeWordCount() const) {
      return elementCount() * (ONE * WORDS / ELEMENTS);
232 233
    }

234
    KJ_ALWAYS_INLINE(void set(ElementSize es, ElementCountN<29> ec)) {
235
      elementSizeAndCount.set(unboundAs<uint32_t>((ec / ELEMENTS) << G(3)) |
236
                              static_cast<int>(es));
237 238
    }

239
    KJ_ALWAYS_INLINE(void setInlineComposite(WordCountN<29> wc)) {
240
      elementSizeAndCount.set(unboundAs<uint32_t>((wc / WORDS) << G(3)) |
241
                              static_cast<int>(ElementSize::INLINE_COMPOSITE));
242 243 244 245 246 247 248 249 250 251 252
    }
  };

  struct FarRef {
    WireValue<SegmentId> segmentId;

    KJ_ALWAYS_INLINE(void set(SegmentId si)) {
      segmentId.set(si);
    }
  };

253 254 255 256 257
  struct CapRef {
    WireValue<uint32_t> index;
    // Index into the message's capability table.
  };

258
  union {
259 260
    uint32_t upper32Bits;

261
    StructRef structRef;
262

263 264 265
    ListRef listRef;

    FarRef farRef;
266 267

    CapRef capRef;
268
  };
269

270
  KJ_ALWAYS_INLINE(bool isNull() const) {
271 272 273 274
    // If the upper 32 bits are zero, this is a pointer to an empty struct.  We consider that to be
    // our "null" value.
    return (offsetAndKind.get() == 0) & (upper32Bits == 0);
  }
275

276
};
277
static_assert(sizeof(WirePointer) == sizeof(word),
278
    "capnp::WirePointer is not exactly one word.  This will probably break everything.");
279
static_assert(unboundAs<size_t>(POINTERS * WORDS_PER_POINTER * BYTES_PER_WORD / BYTES) ==
280
              sizeof(WirePointer),
281
    "WORDS_PER_POINTER is wrong.");
282
static_assert(unboundAs<size_t>(POINTERS * BYTES_PER_POINTER / BYTES) == sizeof(WirePointer),
283
    "BYTES_PER_POINTER is wrong.");
284
static_assert(unboundAs<size_t>(POINTERS * BITS_PER_POINTER / BITS_PER_BYTE / BYTES) ==
285
              sizeof(WirePointer),
286
    "BITS_PER_POINTER is wrong.");
287

288 289 290
namespace {

static const union {
291
  AlignedData<unbound(POINTER_SIZE_IN_WORDS / WORDS)> word;
292 293 294 295 296
  WirePointer pointer;
} zero = {{{0}}};

}  // namespace

297 298
// =======================================================================================

299 300 301 302 303 304 305 306 307 308
namespace {

template <typename T>
struct SegmentAnd {
  SegmentBuilder* segment;
  T value;
};

}  // namespace

309
struct WireHelpers {
310 311 312
#if CAPNP_DEBUG_TYPES
  template <uint64_t maxN, typename T>
  static KJ_ALWAYS_INLINE(
313 314
      kj::Quantity<kj::Bounded<(maxN + 7) / 8, T>, word> roundBytesUpToWords(
          kj::Quantity<kj::Bounded<maxN, T>, byte> bytes)) {
315 316 317 318 319 320
    static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
    return (bytes + G(7) * BYTES) / BYTES_PER_WORD;
  }

  template <uint64_t maxN, typename T>
  static KJ_ALWAYS_INLINE(
321 322
      kj::Quantity<kj::Bounded<(maxN + 7) / 8, T>, byte> roundBitsUpToBytes(
          kj::Quantity<kj::Bounded<maxN, T>, BitLabel> bits)) {
323 324 325 326 327
    return (bits + G(7) * BITS) / BITS_PER_BYTE;
  }

  template <uint64_t maxN, typename T>
  static KJ_ALWAYS_INLINE(
328 329
      kj::Quantity<kj::Bounded<(maxN + 63) / 64, T>, word> roundBitsUpToWords(
          kj::Quantity<kj::Bounded<maxN, T>, BitLabel> bits)) {
330 331 332 333
    static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
    return (bits + G(63) * BITS) / BITS_PER_WORD;
  }
#else
334
  static KJ_ALWAYS_INLINE(WordCount roundBytesUpToWords(ByteCount bytes)) {
Kenton Varda's avatar
Kenton Varda committed
335
    static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
336
    return (bytes + G(7) * BYTES) / BYTES_PER_WORD;
337 338
  }

339
  static KJ_ALWAYS_INLINE(ByteCount roundBitsUpToBytes(BitCount bits)) {
340
    return (bits + G(7) * BITS) / BITS_PER_BYTE;
Kenton Varda's avatar
Kenton Varda committed
341 342
  }

343
  static KJ_ALWAYS_INLINE(WordCount64 roundBitsUpToWords(BitCount64 bits)) {
344
    static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
345
    return (bits + G(63) * BITS) / BITS_PER_WORD;
346 347
  }

348
  static KJ_ALWAYS_INLINE(ByteCount64 roundBitsUpToBytes(BitCount64 bits)) {
349 350 351 352 353
    return (bits + G(7) * BITS) / BITS_PER_BYTE;
  }
#endif

  static KJ_ALWAYS_INLINE(void zeroMemory(byte* ptr, ByteCount32 count)) {
354
    memset(ptr, 0, unbound(count / BYTES));
355 356 357
  }

  static KJ_ALWAYS_INLINE(void zeroMemory(word* ptr, WordCountN<29> count)) {
358
    memset(ptr, 0, unbound(count * BYTES_PER_WORD / BYTES));
359 360 361
  }

  static KJ_ALWAYS_INLINE(void zeroMemory(WirePointer* ptr, WirePointerCountN<29> count)) {
362
    memset(ptr, 0, unbound(count * BYTES_PER_POINTER / BYTES));
363 364
  }

365 366 367 368 369
  static KJ_ALWAYS_INLINE(void zeroMemory(WirePointer* ptr)) {
    memset(ptr, 0, sizeof(*ptr));
  }

  template <typename T>
Kenton Varda's avatar
Kenton Varda committed
370
  static inline void zeroMemory(kj::ArrayPtr<T> array) {
371 372 373
    memset(array.begin(), 0, array.size() * sizeof(array[0]));
  }

374
  static KJ_ALWAYS_INLINE(void copyMemory(byte* to, const byte* from, ByteCount32 count)) {
375
    memcpy(to, from, unbound(count / BYTES));
376 377 378
  }

  static KJ_ALWAYS_INLINE(void copyMemory(word* to, const word* from, WordCountN<29> count)) {
379
    memcpy(to, from, unbound(count * BYTES_PER_WORD / BYTES));
380 381 382 383
  }

  static KJ_ALWAYS_INLINE(void copyMemory(WirePointer* to, const WirePointer* from,
                                          WirePointerCountN<29> count)) {
384
    memcpy(to, from, unbound(count * BYTES_PER_POINTER  / BYTES));
385 386
  }

387
  template <typename T>
Kenton Varda's avatar
Kenton Varda committed
388
  static inline void copyMemory(T* to, const T* from) {
389 390 391 392 393
    memcpy(to, from, sizeof(*from));
  }

  // TODO(cleanup): Turn these into a .copyTo() method of ArrayPtr?
  template <typename T>
Kenton Varda's avatar
Kenton Varda committed
394
  static inline void copyMemory(T* to, kj::ArrayPtr<T> from) {
395 396 397
    memcpy(to, from.begin(), from.size() * sizeof(from[0]));
  }
  template <typename T>
Kenton Varda's avatar
Kenton Varda committed
398
  static inline void copyMemory(T* to, kj::ArrayPtr<const T> from) {
399 400 401 402 403 404
    memcpy(to, from.begin(), from.size() * sizeof(from[0]));
  }
  static KJ_ALWAYS_INLINE(void copyMemory(char* to, kj::StringPtr from)) {
    memcpy(to, from.begin(), from.size() * sizeof(from[0]));
  }

405
  static KJ_ALWAYS_INLINE(bool boundsCheck(
406
      SegmentReader* segment, const word* start, const word* end)) {
407
    // If segment is null, this is an unchecked message, so we don't do bounds checks.
408 409 410
    return segment == nullptr || segment->containsInterval(start, end);
  }

411 412 413 414 415
  static KJ_ALWAYS_INLINE(bool amplifiedRead(SegmentReader* segment, WordCount virtualAmount)) {
    // If segment is null, this is an unchecked message, so we don't do read limiter checks.
    return segment == nullptr || segment->amplifiedRead(virtualAmount);
  }

416
  static KJ_ALWAYS_INLINE(word* allocate(
417 418
      WirePointer*& ref, SegmentBuilder*& segment, CapTableBuilder* capTable,
      SegmentWordCount amount, WirePointer::Kind kind, BuilderArena* orphanArena)) {
David Renshaw's avatar
David Renshaw committed
419
    // Allocate space in the message for a new object, creating far pointers if necessary.
420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
    //
    // * `ref` starts out being a reference to the pointer which shall be assigned to point at the
    //   new object.  On return, `ref` points to a pointer which needs to be initialized with
    //   the object's type information.  Normally this is the same pointer, but it can change if
    //   a far pointer was allocated -- in this case, `ref` will end up pointing to the far
    //   pointer's tag.  Either way, `allocate()` takes care of making sure that the original
    //   pointer ends up leading to the new object.  On return, only the upper 32 bit of `*ref`
    //   need to be filled in by the caller.
    // * `segment` starts out pointing to the segment containing `ref`.  On return, it points to
    //   the segment containing the allocated object, which is usually the same segment but could
    //   be a different one if the original segment was out of space.
    // * `amount` is the number of words to allocate.
    // * `kind` is the kind of object to allocate.  It is used to initialize the pointer.  It
    //   cannot be `FAR` -- far pointers are allocated automatically as needed.
    // * `orphanArena` is usually null.  If it is non-null, then we're allocating an orphan object.
    //   In this case, `segment` starts out null; the allocation takes place in an arbitrary
    //   segment belonging to the arena.  `ref` will be initialized as a non-far pointer, but its
    //   target offset will be set to zero.

439
    if (orphanArena == nullptr) {
440
      if (!ref->isNull()) zeroObject(segment, capTable, ref);
441

442
      if (amount == ZERO * WORDS && kind == WirePointer::STRUCT) {
443 444 445 446 447 448
        // Note that the check for kind == WirePointer::STRUCT will hopefully cause this whole
        // branch to be optimized away from all the call sites that are allocating non-structs.
        ref->setKindAndTargetForEmptyStruct();
        return reinterpret_cast<word*>(ref);
      }

449 450 451
      word* ptr = segment->allocate(amount);

      if (ptr == nullptr) {
Matthew Maurer's avatar
Matthew Maurer committed
452

453 454 455 456
        // Need to allocate in a new segment.  We'll need to allocate an extra pointer worth of
        // space to act as the landing pad for a far pointer.

        WordCount amountPlusRef = amount + POINTER_SIZE_IN_WORDS;
457 458 459 460
        auto allocation = segment->getArena()->allocate(
            assertMaxBits<SEGMENT_WORD_COUNT_BITS>(amountPlusRef, []() {
              KJ_FAIL_REQUIRE("requested object size exceeds maximum segment size");
            }));
461 462 463 464 465 466 467 468 469
        segment = allocation.segment;
        ptr = allocation.words;

        // Set up the original pointer to be a far pointer to the new segment.
        ref->setFar(false, segment->getOffsetTo(ptr));
        ref->farRef.set(segment->getSegmentId());

        // Initialize the landing pad to indicate that the data immediately follows the pad.
        ref = reinterpret_cast<WirePointer*>(ptr);
470
        ref->setKindAndTarget(kind, ptr + POINTER_SIZE_IN_WORDS, segment);
471 472 473 474

        // Allocated space follows new pointer.
        return ptr + POINTER_SIZE_IN_WORDS;
      } else {
475
        ref->setKindAndTarget(kind, ptr, segment);
476 477
        return ptr;
      }
478
    } else {
479
      // orphanArena is non-null.  Allocate an orphan.
480
      KJ_DASSERT(ref->isNull());
481 482
      auto allocation = orphanArena->allocate(amount);
      segment = allocation.segment;
483
      ref->setKindForOrphan(kind);
484
      return allocation.words;
485 486 487
    }
  }

488
  static KJ_ALWAYS_INLINE(word* followFarsNoWritableCheck(
489 490 491 492 493 494 495 496 497 498
      WirePointer*& ref, word* refTarget, SegmentBuilder*& segment)) {
    // If `ref` is a far pointer, follow it.  On return, `ref` will have been updated to point at
    // a WirePointer that contains the type information about the target object, and a pointer to
    // the object contents is returned.  The caller must NOT use `ref->target()` as this may or may
    // not actually return a valid pointer.  `segment` is also updated to point at the segment which
    // actually contains the object.
    //
    // If `ref` is not a far pointer, this simply returns `refTarget`.  Usually, `refTarget` should
    // be the same as `ref->target()`, but may not be in cases where `ref` is only a tag.

499
    if (ref->kind() == WirePointer::FAR) {
500
      segment = segment->getArena()->getSegment(ref->farRef.segmentId.get());
501 502
      WirePointer* pad =
          reinterpret_cast<WirePointer*>(segment->getPtrUnchecked(ref->farPositionInSegment()));
503 504 505
      if (!ref->isDoubleFar()) {
        ref = pad;
        return pad->target();
506
      }
507 508 509 510 511 512 513

      // Landing pad is another far pointer.  It is followed by a tag describing the pointed-to
      // object.
      ref = pad + 1;

      segment = segment->getArena()->getSegment(pad->farRef.segmentId.get());
      return segment->getPtrUnchecked(pad->farPositionInSegment());
514
    } else {
515
      return refTarget;
516 517 518
    }
  }

519 520 521 522 523 524 525
  static KJ_ALWAYS_INLINE(word* followFars(
      WirePointer*& ref, word* refTarget, SegmentBuilder*& segment)) {
    auto result = followFarsNoWritableCheck(ref, refTarget, segment);
    segment->checkWritable();
    return result;
  }

526 527 528
  static KJ_ALWAYS_INLINE(const word* followFars(
      const WirePointer*& ref, const word* refTarget, SegmentReader*& segment)) {
    // Like the other followFars() but operates on readers.
529

530
    // If the segment is null, this is an unchecked message, so there are no FAR pointers.
531
    if (segment != nullptr && ref->kind() == WirePointer::FAR) {
532
      // Look up the segment containing the landing pad.
533
      segment = segment->getArena()->tryGetSegment(ref->farRef.segmentId.get());
534
      KJ_REQUIRE(segment != nullptr, "Message contains far pointer to unknown segment.") {
535
        return nullptr;
Kenton Varda's avatar
Kenton Varda committed
536
      }
537

538 539
      // Find the landing pad and check that it is within bounds.
      const word* ptr = segment->getStartPtr() + ref->farPositionInSegment();
540
      WordCount padWords = bounded(1 + ref->isDoubleFar()) * POINTER_SIZE_IN_WORDS;
541 542
      KJ_REQUIRE(boundsCheck(segment, ptr, ptr + padWords),
                 "Message contains out-of-bounds far pointer.") {
543
        return nullptr;
544 545
      }

546
      const WirePointer* pad = reinterpret_cast<const WirePointer*>(ptr);
547 548 549 550 551 552 553 554 555 556 557

      // If this is not a double-far then the landing pad is our final pointer.
      if (!ref->isDoubleFar()) {
        ref = pad;
        return pad->target();
      }

      // Landing pad is another far pointer.  It is followed by a tag describing the pointed-to
      // object.
      ref = pad + 1;

558 559 560 561 562 563 564
      SegmentReader* newSegment = segment->getArena()->tryGetSegment(pad->farRef.segmentId.get());
      KJ_REQUIRE(newSegment != nullptr,
          "Message contains double-far pointer to unknown segment.") {
        return nullptr;
      }
      KJ_REQUIRE(pad->kind() == WirePointer::FAR,
          "Second word of double-far pad must be far pointer.") {
565
        return nullptr;
566
      }
567

568
      segment = newSegment;
569
      return segment->getStartPtr() + pad->farPositionInSegment();
570
    } else {
571
      return refTarget;
572 573 574
    }
  }

575 576
  // -----------------------------------------------------------------

577
  static void zeroObject(SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref) {
578 579 580
    // Zero out the pointed-to object.  Use when the pointer is about to be overwritten making the
    // target object no longer reachable.

581 582 583
    // We shouldn't zero out external data linked into the message.
    if (!segment->isWritable()) return;

584 585 586
    switch (ref->kind()) {
      case WirePointer::STRUCT:
      case WirePointer::LIST:
587
        zeroObject(segment, capTable, ref, ref->target());
588 589 590
        break;
      case WirePointer::FAR: {
        segment = segment->getArena()->getSegment(ref->farRef.segmentId.get());
591 592 593 594 595 596 597
        if (segment->isWritable()) {  // Don't zero external data.
          WirePointer* pad =
              reinterpret_cast<WirePointer*>(segment->getPtrUnchecked(ref->farPositionInSegment()));

          if (ref->isDoubleFar()) {
            segment = segment->getArena()->getSegment(pad->farRef.segmentId.get());
            if (segment->isWritable()) {
598 599
              zeroObject(segment, capTable,
                         pad + 1, segment->getPtrUnchecked(pad->farPositionInSegment()));
600
            }
601
            zeroMemory(pad, G(2) * POINTERS);
602
          } else {
603
            zeroObject(segment, capTable, pad);
604
            zeroMemory(pad);
605
          }
606 607 608
        }
        break;
      }
609 610
      case WirePointer::OTHER:
        if (ref->isCapability()) {
611 612 613
#if CAPNP_LITE
          KJ_FAIL_ASSERT("Capability encountered in builder in lite mode?") { break; }
#else  // CAPNP_LINE
614
          capTable->dropCap(ref->capRef.index.get());
615
#endif  // CAPNP_LITE, else
616 617 618 619
        } else {
          KJ_FAIL_REQUIRE("Unknown pointer type.") { break; }
        }
        break;
620 621 622
    }
  }

623 624
  static void zeroObject(SegmentBuilder* segment, CapTableBuilder* capTable,
                         WirePointer* tag, word* ptr) {
625 626 627
    // We shouldn't zero out external data linked into the message.
    if (!segment->isWritable()) return;

628 629 630 631
    switch (tag->kind()) {
      case WirePointer::STRUCT: {
        WirePointer* pointerSection =
            reinterpret_cast<WirePointer*>(ptr + tag->structRef.dataSize.get());
632
        for (auto i: kj::zeroTo(tag->structRef.ptrCount.get())) {
633
          zeroObject(segment, capTable, pointerSection + i);
634
        }
635
        zeroMemory(ptr, tag->structRef.wordSize());
636 637 638 639
        break;
      }
      case WirePointer::LIST: {
        switch (tag->listRef.elementSize()) {
640
          case ElementSize::VOID:
641 642
            // Nothing.
            break;
643 644 645 646
          case ElementSize::BIT:
          case ElementSize::BYTE:
          case ElementSize::TWO_BYTES:
          case ElementSize::FOUR_BYTES:
647 648
          case ElementSize::EIGHT_BYTES: {
            zeroMemory(ptr, roundBitsUpToWords(
649
                upgradeBound<uint64_t>(tag->listRef.elementCount()) *
650
                dataBitsPerElement(tag->listRef.elementSize())));
651
            break;
652
          }
653
          case ElementSize::POINTER: {
654 655 656 657
            WirePointer* typedPtr = reinterpret_cast<WirePointer*>(ptr);
            auto count = tag->listRef.elementCount() * (ONE * POINTERS / ELEMENTS);
            for (auto i: kj::zeroTo(count)) {
              zeroObject(segment, capTable, typedPtr + i);
658
            }
659
            zeroMemory(typedPtr, count);
660 661
            break;
          }
662
          case ElementSize::INLINE_COMPOSITE: {
663 664
            WirePointer* elementTag = reinterpret_cast<WirePointer*>(ptr);

665
            KJ_ASSERT(elementTag->kind() == WirePointer::STRUCT,
666 667 668 669
                  "Don't know how to handle non-STRUCT inline composite.");
            WordCount dataSize = elementTag->structRef.dataSize.get();
            WirePointerCount pointerCount = elementTag->structRef.ptrCount.get();

670
            auto count = elementTag->inlineCompositeListElementCount();
671
            if (pointerCount > ZERO * POINTERS) {
672
              word* pos = ptr + POINTER_SIZE_IN_WORDS;
673
              for (auto i KJ_UNUSED: kj::zeroTo(count)) {
674 675
                pos += dataSize;

676
                for (auto j KJ_UNUSED: kj::zeroTo(pointerCount)) {
677
                  zeroObject(segment, capTable, reinterpret_cast<WirePointer*>(pos));
678 679
                  pos += POINTER_SIZE_IN_WORDS;
                }
680 681 682
              }
            }

683 684
            auto wordsPerElement = elementTag->structRef.wordSize() / ELEMENTS;
            zeroMemory(ptr, assertMaxBits<SEGMENT_WORD_COUNT_BITS>(POINTER_SIZE_IN_WORDS +
685
                upgradeBound<uint64_t>(count) * wordsPerElement, []() {
686 687 688
                  KJ_FAIL_ASSERT("encountered list pointer in builder which is too large to "
                      "possibly fit in a segment. Bug in builder code?");
                }));
689 690 691 692 693 694
            break;
          }
        }
        break;
      }
      case WirePointer::FAR:
695 696 697
        KJ_FAIL_ASSERT("Unexpected FAR pointer.") {
          break;
        }
698
        break;
699 700 701 702 703
      case WirePointer::OTHER:
        KJ_FAIL_ASSERT("Unexpected OTHER pointer.") {
          break;
        }
        break;
704 705 706
    }
  }

707
  static KJ_ALWAYS_INLINE(
708 709 710 711 712
      void zeroPointerAndFars(SegmentBuilder* segment, WirePointer* ref)) {
    // Zero out the pointer itself and, if it is a far pointer, zero the landing pad as well, but
    // do not zero the object body.  Used when upgrading.

    if (ref->kind() == WirePointer::FAR) {
713 714
      SegmentBuilder* padSegment = segment->getArena()->getSegment(ref->farRef.segmentId.get());
      if (padSegment->isWritable()) {  // Don't zero external data.
715 716 717 718 719 720 721
        WirePointer* pad = reinterpret_cast<WirePointer*>(
            padSegment->getPtrUnchecked(ref->farPositionInSegment()));
        if (ref->isDoubleFar()) {
          zeroMemory(pad, G(2) * POINTERS);
        } else {
          zeroMemory(pad);
        }
722
      }
723
    }
724 725

    zeroMemory(ref);
726 727
  }

728 729 730

  // -----------------------------------------------------------------

731 732
  static MessageSizeCounts totalSize(
      SegmentReader* segment, const WirePointer* ref, int nestingLimit) {
733 734
    // Compute the total size of the object pointed to, not counting far pointer overhead.

735
    MessageSizeCounts result = { ZERO * WORDS, 0 };
736

737
    if (ref->isNull()) {
738
      return result;
739 740
    }

741
    KJ_REQUIRE(nestingLimit > 0, "Message is too deeply-nested.") {
742
      return result;
743 744 745
    }
    --nestingLimit;

746
    const word* ptr = followFars(ref, ref->target(), segment);
747 748

    switch (ref->kind()) {
749
      case WirePointer::STRUCT: {
750 751 752
        KJ_REQUIRE(boundsCheck(segment, ptr, ptr + ref->structRef.wordSize()),
                   "Message contained out-of-bounds struct pointer.") {
          return result;
753
        }
754
        result.addWords(ref->structRef.wordSize());
755 756 757

        const WirePointer* pointerSection =
            reinterpret_cast<const WirePointer*>(ptr + ref->structRef.dataSize.get());
758
        for (auto i: kj::zeroTo(ref->structRef.ptrCount.get())) {
759 760 761 762 763 764
          result += totalSize(segment, pointerSection + i, nestingLimit);
        }
        break;
      }
      case WirePointer::LIST: {
        switch (ref->listRef.elementSize()) {
765
          case ElementSize::VOID:
766 767
            // Nothing.
            break;
768 769 770 771 772
          case ElementSize::BIT:
          case ElementSize::BYTE:
          case ElementSize::TWO_BYTES:
          case ElementSize::FOUR_BYTES:
          case ElementSize::EIGHT_BYTES: {
773
            auto totalWords = roundBitsUpToWords(
774
                upgradeBound<uint64_t>(ref->listRef.elementCount()) *
775
                dataBitsPerElement(ref->listRef.elementSize()));
776 777 778
            KJ_REQUIRE(boundsCheck(segment, ptr, ptr + totalWords),
                       "Message contained out-of-bounds list pointer.") {
              return result;
779
            }
780
            result.addWords(totalWords);
781 782
            break;
          }
783
          case ElementSize::POINTER: {
784 785
            WirePointerCount count = ref->listRef.elementCount() * (POINTERS / ELEMENTS);

786 787 788
            KJ_REQUIRE(boundsCheck(segment, ptr, ptr + count * WORDS_PER_POINTER),
                       "Message contained out-of-bounds list pointer.") {
              return result;
789 790
            }

791
            result.addWords(count * WORDS_PER_POINTER);
792

793
            for (auto i: kj::zeroTo(count)) {
794 795 796 797 798
              result += totalSize(segment, reinterpret_cast<const WirePointer*>(ptr) + i,
                                  nestingLimit);
            }
            break;
          }
799
          case ElementSize::INLINE_COMPOSITE: {
800
            auto wordCount = ref->listRef.inlineCompositeWordCount();
801 802 803
            KJ_REQUIRE(boundsCheck(segment, ptr, ptr + wordCount + POINTER_SIZE_IN_WORDS),
                       "Message contained out-of-bounds list pointer.") {
              return result;
804 805 806
            }

            const WirePointer* elementTag = reinterpret_cast<const WirePointer*>(ptr);
807
            auto count = elementTag->inlineCompositeListElementCount();
808

809 810 811
            KJ_REQUIRE(elementTag->kind() == WirePointer::STRUCT,
                       "Don't know how to handle non-STRUCT inline composite.") {
              return result;
812
            }
813

814
            auto actualSize = elementTag->structRef.wordSize() / ELEMENTS *
815
                              upgradeBound<uint64_t>(count);
816
            KJ_REQUIRE(actualSize <= wordCount,
817 818
                       "Struct list pointer's elements overran size.") {
              return result;
819 820
            }

821 822
            // We count the actual size rather than the claimed word count because that's what
            // we'll end up with if we make a copy.
823
            result.addWords(wordCount + POINTER_SIZE_IN_WORDS);
824

825 826 827
            WordCount dataSize = elementTag->structRef.dataSize.get();
            WirePointerCount pointerCount = elementTag->structRef.ptrCount.get();

828
            if (pointerCount > ZERO * POINTERS) {
829
              const word* pos = ptr + POINTER_SIZE_IN_WORDS;
830
              for (auto i KJ_UNUSED: kj::zeroTo(count)) {
831
                pos += dataSize;
832

833
                for (auto j KJ_UNUSED: kj::zeroTo(pointerCount)) {
834 835 836 837
                  result += totalSize(segment, reinterpret_cast<const WirePointer*>(pos),
                                      nestingLimit);
                  pos += POINTER_SIZE_IN_WORDS;
                }
838 839 840 841 842 843 844 845
              }
            }
            break;
          }
        }
        break;
      }
      case WirePointer::FAR:
846
        KJ_FAIL_REQUIRE("Unexpected FAR pointer.") {
847 848 849
          break;
        }
        break;
850
      case WirePointer::OTHER:
851 852 853 854 855
        if (ref->isCapability()) {
          result.capCount++;
        } else {
          KJ_FAIL_REQUIRE("Unknown pointer type.") { break; }
        }
856
        break;
857 858 859 860 861
    }

    return result;
  }

862
  // -----------------------------------------------------------------
863
  // Copy from an unchecked message.
864

865
  static KJ_ALWAYS_INLINE(
866 867
      void copyStruct(SegmentBuilder* segment, CapTableBuilder* capTable,
                      word* dst, const word* src,
868 869
                      StructDataWordCount dataSize, StructPointerCount pointerCount)) {
    copyMemory(dst, src, dataSize);
870

871 872
    const WirePointer* srcRefs = reinterpret_cast<const WirePointer*>(src + dataSize);
    WirePointer* dstRefs = reinterpret_cast<WirePointer*>(dst + dataSize);
873

874
    for (auto i: kj::zeroTo(pointerCount)) {
875
      SegmentBuilder* subSegment = segment;
876
      WirePointer* dstRef = dstRefs + i;
877
      copyMessage(subSegment, capTable, dstRef, srcRefs + i);
878 879 880
    }
  }

881
  static word* copyMessage(
882 883
      SegmentBuilder*& segment, CapTableBuilder* capTable,
      WirePointer*& dst, const WirePointer* src) {
884 885
    // Not always-inline because it's recursive.

886
    switch (src->kind()) {
887
      case WirePointer::STRUCT: {
888
        if (src->isNull()) {
889
          zeroMemory(dst);
890
          return nullptr;
891 892
        } else {
          const word* srcPtr = src->target();
893
          word* dstPtr = allocate(
894
              dst, segment, capTable, src->structRef.wordSize(), WirePointer::STRUCT, nullptr);
895

896
          copyStruct(segment, capTable, dstPtr, srcPtr, src->structRef.dataSize.get(),
897
                     src->structRef.ptrCount.get());
898

899
          dst->structRef.set(src->structRef.dataSize.get(), src->structRef.ptrCount.get());
900
          return dstPtr;
901 902
        }
      }
903
      case WirePointer::LIST: {
904
        switch (src->listRef.elementSize()) {
905 906 907 908 909 910
          case ElementSize::VOID:
          case ElementSize::BIT:
          case ElementSize::BYTE:
          case ElementSize::TWO_BYTES:
          case ElementSize::FOUR_BYTES:
          case ElementSize::EIGHT_BYTES: {
911
            auto wordCount = roundBitsUpToWords(
912
                upgradeBound<uint64_t>(src->listRef.elementCount()) *
913
                dataBitsPerElement(src->listRef.elementSize()));
914
            const word* srcPtr = src->target();
915
            word* dstPtr = allocate(dst, segment, capTable, wordCount, WirePointer::LIST, nullptr);
916
            copyMemory(dstPtr, srcPtr, wordCount);
917

918 919
            dst->listRef.set(src->listRef.elementSize(), src->listRef.elementCount());
            return dstPtr;
920 921
          }

922
          case ElementSize::POINTER: {
923 924
            const WirePointer* srcRefs = reinterpret_cast<const WirePointer*>(src->target());
            WirePointer* dstRefs = reinterpret_cast<WirePointer*>(
925
                allocate(dst, segment, capTable, src->listRef.elementCount() *
926
                    (ONE * POINTERS / ELEMENTS) * WORDS_PER_POINTER,
927
                    WirePointer::LIST, nullptr));
928

929
            for (auto i: kj::zeroTo(src->listRef.elementCount() * (ONE * POINTERS / ELEMENTS))) {
930
              SegmentBuilder* subSegment = segment;
931
              WirePointer* dstRef = dstRefs + i;
932
              copyMessage(subSegment, capTable, dstRef, srcRefs + i);
933 934
            }

935
            dst->listRef.set(ElementSize::POINTER, src->listRef.elementCount());
936
            return reinterpret_cast<word*>(dstRefs);
937 938
          }

939
          case ElementSize::INLINE_COMPOSITE: {
940
            const word* srcPtr = src->target();
941
            word* dstPtr = allocate(dst, segment, capTable,
942 943 944
                assertMaxBits<SEGMENT_WORD_COUNT_BITS>(
                    src->listRef.inlineCompositeWordCount() + POINTER_SIZE_IN_WORDS,
                    []() { KJ_FAIL_ASSERT("list too big to fit in a segment"); }),
945
                WirePointer::LIST, nullptr);
946

947
            dst->listRef.setInlineComposite(src->listRef.inlineCompositeWordCount());
948

949
            const WirePointer* srcTag = reinterpret_cast<const WirePointer*>(srcPtr);
950
            copyMemory(reinterpret_cast<WirePointer*>(dstPtr), srcTag);
951

952 953
            const word* srcElement = srcPtr + POINTER_SIZE_IN_WORDS;
            word* dstElement = dstPtr + POINTER_SIZE_IN_WORDS;
954

955
            KJ_ASSERT(srcTag->kind() == WirePointer::STRUCT,
956 957
                "INLINE_COMPOSITE of lists is not yet supported.");

958
            for (auto i KJ_UNUSED: kj::zeroTo(srcTag->inlineCompositeListElementCount())) {
959
              copyStruct(segment, capTable, dstElement, srcElement,
960
                  srcTag->structRef.dataSize.get(), srcTag->structRef.ptrCount.get());
961 962
              srcElement += srcTag->structRef.wordSize();
              dstElement += srcTag->structRef.wordSize();
963
            }
964
            return dstPtr;
965 966 967 968
          }
        }
        break;
      }
969 970
      case WirePointer::OTHER:
        KJ_FAIL_REQUIRE("Unchecked messages cannot contain OTHER pointers (e.g. capabilities).");
971 972 973
        break;
      case WirePointer::FAR:
        KJ_FAIL_REQUIRE("Unchecked messages cannot contain far pointers.");
974 975 976
        break;
    }

977
    return nullptr;
978 979
  }

980 981
  static void transferPointer(SegmentBuilder* dstSegment, WirePointer* dst,
                              SegmentBuilder* srcSegment, WirePointer* src) {
982 983
    // Make *dst point to the same object as *src.  Both must reside in the same message, but can
    // be in different segments.  Not always-inline because this is rarely used.
984 985 986 987 988
    //
    // Caller MUST zero out the source pointer after calling this, to make sure no later code
    // mistakenly thinks the source location still owns the object.  transferPointer() doesn't do
    // this zeroing itself because many callers transfer several pointers in a loop then zero out
    // the whole section.
989 990 991 992

    KJ_DASSERT(dst->isNull());
    // We expect the caller to ensure the target is already null so won't leak.

993
    if (src->isNull()) {
994
      zeroMemory(dst);
995
    } else if (src->isPositional()) {
996
      transferPointer(dstSegment, dst, srcSegment, src, src->target());
997 998
    } else {
      // Far and other pointers are position-independent, so we can just copy.
999
      copyMemory(dst, src);
1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
    }
  }

  static void transferPointer(SegmentBuilder* dstSegment, WirePointer* dst,
                              SegmentBuilder* srcSegment, const WirePointer* srcTag,
                              word* srcPtr) {
    // Like the other overload, but splits src into a tag and a target.  Particularly useful for
    // OrphanBuilder.

    if (dstSegment == srcSegment) {
1010
      // Same segment, so create a direct pointer.
1011

1012
      if (srcTag->kind() == WirePointer::STRUCT && srcTag->structRef.wordSize() == ZERO * WORDS) {
1013 1014 1015 1016
        dst->setKindAndTargetForEmptyStruct();
      } else {
        dst->setKindAndTarget(srcTag->kind(), srcPtr, dstSegment);
      }
1017 1018

      // We can just copy the upper 32 bits.  (Use memcpy() to comply with aliasing rules.)
1019
      copyMemory(&dst->upper32Bits, &srcTag->upper32Bits);
1020 1021 1022 1023
    } else {
      // Need to create a far pointer.  Try to allocate it in the same segment as the source, so
      // that it doesn't need to be a double-far.

1024
      WirePointer* landingPad =
1025
          reinterpret_cast<WirePointer*>(srcSegment->allocate(G(1) * WORDS));
1026 1027
      if (landingPad == nullptr) {
        // Darn, need a double-far.
1028
        auto allocation = srcSegment->getArena()->allocate(G(2) * WORDS);
1029 1030
        SegmentBuilder* farSegment = allocation.segment;
        landingPad = reinterpret_cast<WirePointer*>(allocation.words);
1031

1032
        landingPad[0].setFar(false, srcSegment->getOffsetTo(srcPtr));
1033 1034
        landingPad[0].farRef.segmentId.set(srcSegment->getSegmentId());

1035
        landingPad[1].setKindWithZeroOffset(srcTag->kind());
1036
        copyMemory(&landingPad[1].upper32Bits, &srcTag->upper32Bits);
1037 1038 1039 1040 1041

        dst->setFar(true, farSegment->getOffsetTo(reinterpret_cast<word*>(landingPad)));
        dst->farRef.set(farSegment->getSegmentId());
      } else {
        // Simple landing pad is just a pointer.
1042
        landingPad->setKindAndTarget(srcTag->kind(), srcPtr, srcSegment);
1043
        copyMemory(&landingPad->upper32Bits, &srcTag->upper32Bits);
1044 1045 1046 1047 1048 1049 1050

        dst->setFar(false, srcSegment->getOffsetTo(reinterpret_cast<word*>(landingPad)));
        dst->farRef.set(srcSegment->getSegmentId());
      }
    }
  }

1051 1052
  // -----------------------------------------------------------------

1053
  static KJ_ALWAYS_INLINE(StructBuilder initStructPointer(
1054
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, StructSize size,
1055
      BuilderArena* orphanArena = nullptr)) {
1056
    // Allocate space for the new struct.  Newly-allocated space is automatically zeroed.
1057
    word* ptr = allocate(ref, segment, capTable, size.total(), WirePointer::STRUCT, orphanArena);
1058

1059
    // Initialize the pointer.
1060
    ref->structRef.set(size);
1061 1062

    // Build the StructBuilder.
1063
    return StructBuilder(segment, capTable, ptr, reinterpret_cast<WirePointer*>(ptr + size.data),
1064
                         size.data * BITS_PER_WORD, size.pointers);
1065
  }
1066

1067
  static KJ_ALWAYS_INLINE(StructBuilder getWritableStructPointer(
1068 1069 1070
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, StructSize size,
      const word* defaultValue)) {
    return getWritableStructPointer(ref, ref->target(), segment, capTable, size, defaultValue);
1071 1072 1073
  }

  static KJ_ALWAYS_INLINE(StructBuilder getWritableStructPointer(
1074 1075
      WirePointer* ref, word* refTarget, SegmentBuilder* segment, CapTableBuilder* capTable,
      StructSize size, const word* defaultValue, BuilderArena* orphanArena = nullptr)) {
1076
    if (ref->isNull()) {
1077
    useDefault:
Kenton Varda's avatar
Kenton Varda committed
1078
      if (defaultValue == nullptr ||
1079
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
1080
        return initStructPointer(ref, segment, capTable, size, orphanArena);
1081
      }
1082 1083
      refTarget = copyMessage(segment, capTable, ref,
          reinterpret_cast<const WirePointer*>(defaultValue));
1084 1085
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }
1086

1087 1088
    WirePointer* oldRef = ref;
    SegmentBuilder* oldSegment = segment;
1089
    word* oldPtr = followFars(oldRef, refTarget, oldSegment);
1090

1091
    KJ_REQUIRE(oldRef->kind() == WirePointer::STRUCT,
1092 1093 1094
        "Message contains non-struct pointer where struct pointer was expected.") {
      goto useDefault;
    }
1095

1096 1097
    auto oldDataSize = oldRef->structRef.dataSize.get();
    auto oldPointerCount = oldRef->structRef.ptrCount.get();
1098 1099
    WirePointer* oldPointerSection =
        reinterpret_cast<WirePointer*>(oldPtr + oldDataSize);
1100

1101 1102 1103 1104
    if (oldDataSize < size.data || oldPointerCount < size.pointers) {
      // The space allocated for this struct is too small.  Unlike with readers, we can't just
      // run with it and do bounds checks at access time, because how would we handle writes?
      // Instead, we have to copy the struct to a new space now.
1105

1106 1107 1108
      auto newDataSize = kj::max(oldDataSize, size.data);
      auto newPointerCount = kj::max(oldPointerCount, size.pointers);
      auto totalSize = newDataSize + newPointerCount * WORDS_PER_POINTER;
1109

1110 1111
      // Don't let allocate() zero out the object just yet.
      zeroPointerAndFars(segment, ref);
1112

1113
      word* ptr = allocate(ref, segment, capTable, totalSize, WirePointer::STRUCT, orphanArena);
1114
      ref->structRef.set(newDataSize, newPointerCount);
1115

1116
      // Copy data section.
1117
      copyMemory(ptr, oldPtr, oldDataSize);
1118

1119 1120
      // Copy pointer section.
      WirePointer* newPointerSection = reinterpret_cast<WirePointer*>(ptr + newDataSize);
1121
      for (auto i: kj::zeroTo(oldPointerCount)) {
1122
        transferPointer(segment, newPointerSection + i, oldSegment, oldPointerSection + i);
1123
      }
1124 1125 1126 1127 1128 1129

      // Zero out old location.  This has two purposes:
      // 1) We don't want to leak the original contents of the struct when the message is written
      //    out as it may contain secrets that the caller intends to remove from the new copy.
      // 2) Zeros will be deflated by packing, making this dead memory almost-free if it ever
      //    hits the wire.
1130
      zeroMemory(oldPtr, oldDataSize + oldPointerCount * WORDS_PER_POINTER);
1131

1132
      return StructBuilder(segment, capTable, ptr, newPointerSection, newDataSize * BITS_PER_WORD,
1133
                           newPointerCount);
1134
    } else {
1135 1136
      return StructBuilder(oldSegment, capTable, oldPtr, oldPointerSection,
                           oldDataSize * BITS_PER_WORD, oldPointerCount);
1137
    }
1138 1139
  }

1140
  static KJ_ALWAYS_INLINE(ListBuilder initListPointer(
1141 1142
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
      ElementCount elementCount, ElementSize elementSize, BuilderArena* orphanArena = nullptr)) {
1143
    KJ_DREQUIRE(elementSize != ElementSize::INLINE_COMPOSITE,
1144
        "Should have called initStructListPointer() instead.");
1145

1146 1147 1148 1149 1150 1151 1152
    auto checkedElementCount = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(elementCount,
        []() { KJ_FAIL_REQUIRE("tried to allocate list with too many elements"); });

    auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
    auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
    auto step = bitsPerElementIncludingPointers(elementSize);
    KJ_DASSERT(step * ELEMENTS == (dataSize + pointerCount * BITS_PER_POINTER));
1153

1154
    // Calculate size of the list.
1155
    auto wordCount = roundBitsUpToWords(upgradeBound<uint64_t>(checkedElementCount) * step);
1156

1157
    // Allocate the list.
1158
    word* ptr = allocate(ref, segment, capTable, wordCount, WirePointer::LIST, orphanArena);
1159

1160
    // Initialize the pointer.
1161
    ref->listRef.set(elementSize, checkedElementCount);
1162

1163
    // Build the ListBuilder.
1164 1165
    return ListBuilder(segment, capTable, ptr, step, checkedElementCount,
                       dataSize, pointerCount, elementSize);
1166
  }
1167

1168
  static KJ_ALWAYS_INLINE(ListBuilder initStructListPointer(
1169 1170
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
      ElementCount elementCount, StructSize elementSize, BuilderArena* orphanArena = nullptr)) {
1171 1172 1173 1174
    auto checkedElementCount = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(elementCount,
        []() { KJ_FAIL_REQUIRE("tried to allocate list with too many elements"); });

    WordsPerElementN<17> wordsPerElement = elementSize.total() / ELEMENTS;
1175

1176
    // Allocate the list, prefixed by a single WirePointer.
1177
    auto wordCount = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1178
        upgradeBound<uint64_t>(checkedElementCount) * wordsPerElement,
1179
        []() { KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size"); });
1180 1181
    word* ptr = allocate(ref, segment, capTable, POINTER_SIZE_IN_WORDS + wordCount,
                         WirePointer::LIST, orphanArena);
1182

1183
    // Initialize the pointer.
1184
    // INLINE_COMPOSITE lists replace the element count with the word count.
1185
    ref->listRef.setInlineComposite(wordCount);
1186

1187
    // Initialize the list tag.
1188
    reinterpret_cast<WirePointer*>(ptr)->setKindAndInlineCompositeListElementCount(
1189
        WirePointer::STRUCT, checkedElementCount);
1190 1191
    reinterpret_cast<WirePointer*>(ptr)->structRef.set(elementSize);
    ptr += POINTER_SIZE_IN_WORDS;
1192

1193
    // Build the ListBuilder.
1194
    return ListBuilder(segment, capTable, ptr, wordsPerElement * BITS_PER_WORD, checkedElementCount,
1195
                       elementSize.data * BITS_PER_WORD, elementSize.pointers,
1196
                       ElementSize::INLINE_COMPOSITE);
1197 1198
  }

1199
  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointer(
1200 1201 1202
      WirePointer* origRef, SegmentBuilder* origSegment, CapTableBuilder* capTable,
      ElementSize elementSize, const word* defaultValue)) {
    return getWritableListPointer(origRef, origRef->target(), origSegment, capTable, elementSize,
1203 1204 1205 1206
                                  defaultValue);
  }

  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointer(
1207 1208
      WirePointer* origRef, word* origRefTarget,
      SegmentBuilder* origSegment, CapTableBuilder* capTable, ElementSize elementSize,
1209
      const word* defaultValue, BuilderArena* orphanArena = nullptr)) {
1210
    KJ_DREQUIRE(elementSize != ElementSize::INLINE_COMPOSITE,
1211
             "Use getStructList{Element,Field}() for structs.");
1212

1213 1214
    if (origRef->isNull()) {
    useDefault:
Kenton Varda's avatar
Kenton Varda committed
1215
      if (defaultValue == nullptr ||
1216
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
1217
        return ListBuilder(elementSize);
1218
      }
1219
      origRefTarget = copyMessage(
1220
          origSegment, capTable, origRef, reinterpret_cast<const WirePointer*>(defaultValue));
1221 1222
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }
1223

1224
    // We must verify that the pointer has the right size.  Unlike in
David Renshaw's avatar
David Renshaw committed
1225
    // getWritableStructListPointer(), we never need to "upgrade" the data, because this
1226 1227
    // method is called only for non-struct lists, and there is no allowed upgrade path *to*
    // a non-struct list, only *from* them.
1228

1229 1230
    WirePointer* ref = origRef;
    SegmentBuilder* segment = origSegment;
1231
    word* ptr = followFars(ref, origRefTarget, segment);
1232

1233
    KJ_REQUIRE(ref->kind() == WirePointer::LIST,
1234 1235 1236
        "Called getList{Field,Element}() but existing pointer is not a list.") {
      goto useDefault;
    }
1237

1238
    ElementSize oldSize = ref->listRef.elementSize();
1239

1240
    if (oldSize == ElementSize::INLINE_COMPOSITE) {
1241 1242 1243 1244
      // The existing element size is INLINE_COMPOSITE, though we expected a list of primitives.
      // The existing data must have been written with a newer version of the protocol.  We
      // therefore never need to upgrade the data in this case, but we do need to validate that it
      // is a valid upgrade from what we expected.
1245

1246 1247
      // Read the tag to get the actual element count.
      WirePointer* tag = reinterpret_cast<WirePointer*>(ptr);
1248
      KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
1249 1250
          "INLINE_COMPOSITE list with non-STRUCT elements not supported.");
      ptr += POINTER_SIZE_IN_WORDS;
1251

1252 1253
      auto dataSize = tag->structRef.dataSize.get();
      auto pointerCount = tag->structRef.ptrCount.get();
1254

1255
      switch (elementSize) {
1256
        case ElementSize::VOID:
1257 1258
          // Anything is a valid upgrade from Void.
          break;
1259

1260
        case ElementSize::BIT:
1261 1262 1263 1264 1265 1266 1267
          KJ_FAIL_REQUIRE(
              "Found struct list where bit list was expected; upgrading boolean lists to structs "
              "is no longer supported.") {
            goto useDefault;
          }
          break;

1268 1269 1270 1271
        case ElementSize::BYTE:
        case ElementSize::TWO_BYTES:
        case ElementSize::FOUR_BYTES:
        case ElementSize::EIGHT_BYTES:
1272
          KJ_REQUIRE(dataSize >= ONE * WORDS,
1273 1274 1275
                     "Existing list value is incompatible with expected type.") {
            goto useDefault;
          }
1276
          break;
1277

1278
        case ElementSize::POINTER:
1279
          KJ_REQUIRE(pointerCount >= ONE * POINTERS,
1280 1281 1282
                     "Existing list value is incompatible with expected type.") {
            goto useDefault;
          }
1283 1284 1285
          // Adjust the pointer to point at the reference segment.
          ptr += dataSize;
          break;
1286

1287
        case ElementSize::INLINE_COMPOSITE:
1288
          KJ_UNREACHABLE;
1289
      }
1290

1291
      // OK, looks valid.
1292

1293
      return ListBuilder(segment, capTable, ptr,
1294 1295
                         tag->structRef.wordSize() * BITS_PER_WORD / ELEMENTS,
                         tag->inlineCompositeListElementCount(),
1296
                         dataSize * BITS_PER_WORD, pointerCount, ElementSize::INLINE_COMPOSITE);
1297
    } else {
1298 1299
      auto dataSize = dataBitsPerElement(oldSize) * ELEMENTS;
      auto pointerCount = pointersPerElement(oldSize) * ELEMENTS;
1300

1301 1302
      if (elementSize == ElementSize::BIT) {
        KJ_REQUIRE(oldSize == ElementSize::BIT,
1303 1304 1305 1306
            "Found non-bit list where bit list was expected.") {
          goto useDefault;
        }
      } else {
1307
        KJ_REQUIRE(oldSize != ElementSize::BIT,
1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
            "Found bit list where non-bit list was expected.") {
          goto useDefault;
        }
        KJ_REQUIRE(dataSize >= dataBitsPerElement(elementSize) * ELEMENTS,
                   "Existing list value is incompatible with expected type.") {
          goto useDefault;
        }
        KJ_REQUIRE(pointerCount >= pointersPerElement(elementSize) * ELEMENTS,
                   "Existing list value is incompatible with expected type.") {
          goto useDefault;
        }
1319
      }
1320

1321
      auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
1322
      return ListBuilder(segment, capTable, ptr, step, ref->listRef.elementCount(),
1323
                         dataSize, pointerCount, oldSize);
1324
    }
1325 1326
  }

1327
  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointerAnySize(
1328 1329 1330 1331
      WirePointer* origRef, SegmentBuilder* origSegment, CapTableBuilder* capTable,
      const word* defaultValue)) {
    return getWritableListPointerAnySize(origRef, origRef->target(), origSegment,
                                         capTable, defaultValue);
1332 1333 1334
  }

  static KJ_ALWAYS_INLINE(ListBuilder getWritableListPointerAnySize(
1335 1336
      WirePointer* origRef, word* origRefTarget,
      SegmentBuilder* origSegment, CapTableBuilder* capTable,
1337 1338 1339 1340 1341
      const word* defaultValue, BuilderArena* orphanArena = nullptr)) {
    if (origRef->isNull()) {
    useDefault:
      if (defaultValue == nullptr ||
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
1342
        return ListBuilder(ElementSize::VOID);
1343 1344
      }
      origRefTarget = copyMessage(
1345
          origSegment, capTable, origRef, reinterpret_cast<const WirePointer*>(defaultValue));
1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }

    WirePointer* ref = origRef;
    SegmentBuilder* segment = origSegment;
    word* ptr = followFars(ref, origRefTarget, segment);

    KJ_REQUIRE(ref->kind() == WirePointer::LIST,
        "Called getList{Field,Element}() but existing pointer is not a list.") {
      goto useDefault;
    }

1358
    ElementSize elementSize = ref->listRef.elementSize();
1359

1360
    if (elementSize == ElementSize::INLINE_COMPOSITE) {
1361 1362 1363 1364 1365 1366
      // Read the tag to get the actual element count.
      WirePointer* tag = reinterpret_cast<WirePointer*>(ptr);
      KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
          "INLINE_COMPOSITE list with non-STRUCT elements not supported.");
      ptr += POINTER_SIZE_IN_WORDS;

1367
      return ListBuilder(segment, capTable, ptr,
1368 1369 1370
                         tag->structRef.wordSize() * BITS_PER_WORD / ELEMENTS,
                         tag->inlineCompositeListElementCount(),
                         tag->structRef.dataSize.get() * BITS_PER_WORD,
1371
                         tag->structRef.ptrCount.get(), ElementSize::INLINE_COMPOSITE);
1372
    } else {
1373 1374
      auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
      auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
1375 1376

      auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
1377
      return ListBuilder(segment, capTable, ptr, step, ref->listRef.elementCount(),
1378 1379 1380 1381
                         dataSize, pointerCount, elementSize);
    }
  }

1382
  static KJ_ALWAYS_INLINE(ListBuilder getWritableStructListPointer(
1383 1384 1385 1386
      WirePointer* origRef, SegmentBuilder* origSegment, CapTableBuilder* capTable,
      StructSize elementSize, const word* defaultValue)) {
    return getWritableStructListPointer(origRef, origRef->target(), origSegment, capTable,
                                        elementSize, defaultValue);
1387 1388
  }
  static KJ_ALWAYS_INLINE(ListBuilder getWritableStructListPointer(
1389 1390
      WirePointer* origRef, word* origRefTarget,
      SegmentBuilder* origSegment, CapTableBuilder* capTable,
1391
      StructSize elementSize, const word* defaultValue, BuilderArena* orphanArena = nullptr)) {
1392 1393 1394 1395
    if (origRef->isNull()) {
    useDefault:
      if (defaultValue == nullptr ||
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
1396
        return ListBuilder(ElementSize::INLINE_COMPOSITE);
1397
      }
1398
      origRefTarget = copyMessage(
1399
          origSegment, capTable, origRef, reinterpret_cast<const WirePointer*>(defaultValue));
1400 1401
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }
1402

1403
    // We must verify that the pointer has the right size and potentially upgrade it if not.
1404

1405 1406
    WirePointer* oldRef = origRef;
    SegmentBuilder* oldSegment = origSegment;
1407
    word* oldPtr = followFars(oldRef, origRefTarget, oldSegment);
1408

1409 1410
    KJ_REQUIRE(oldRef->kind() == WirePointer::LIST,
               "Called getList{Field,Element}() but existing pointer is not a list.") {
1411 1412 1413
      goto useDefault;
    }

1414
    ElementSize oldSize = oldRef->listRef.elementSize();
1415

1416
    if (oldSize == ElementSize::INLINE_COMPOSITE) {
1417
      // Existing list is INLINE_COMPOSITE, but we need to verify that the sizes match.
1418

1419 1420
      WirePointer* oldTag = reinterpret_cast<WirePointer*>(oldPtr);
      oldPtr += POINTER_SIZE_IN_WORDS;
1421 1422
      KJ_REQUIRE(oldTag->kind() == WirePointer::STRUCT,
                 "INLINE_COMPOSITE list with non-STRUCT elements not supported.") {
1423 1424 1425
        goto useDefault;
      }

1426 1427
      auto oldDataSize = oldTag->structRef.dataSize.get();
      auto oldPointerCount = oldTag->structRef.ptrCount.get();
1428
      auto oldStep = (oldDataSize + oldPointerCount * WORDS_PER_POINTER) / ELEMENTS;
1429 1430

      auto elementCount = oldTag->inlineCompositeListElementCount();
1431

1432 1433
      if (oldDataSize >= elementSize.data && oldPointerCount >= elementSize.pointers) {
        // Old size is at least as large as we need.  Ship it.
1434
        return ListBuilder(oldSegment, capTable, oldPtr, oldStep * BITS_PER_WORD, elementCount,
1435
                           oldDataSize * BITS_PER_WORD, oldPointerCount,
1436
                           ElementSize::INLINE_COMPOSITE);
1437
      }
1438

1439 1440
      // The structs in this list are smaller than expected, probably written using an older
      // version of the protocol.  We need to make a copy and expand them.
1441

1442 1443
      auto newDataSize = kj::max(oldDataSize, elementSize.data);
      auto newPointerCount = kj::max(oldPointerCount, elementSize.pointers);
1444
      auto newStep = (newDataSize + newPointerCount * WORDS_PER_POINTER) / ELEMENTS;
1445 1446

      auto totalSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1447
            newStep * upgradeBound<uint64_t>(elementCount),
1448
            []() { KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size"); });
1449

1450 1451
      // Don't let allocate() zero out the object just yet.
      zeroPointerAndFars(origSegment, origRef);
1452

1453
      word* newPtr = allocate(origRef, origSegment, capTable, totalSize + POINTER_SIZE_IN_WORDS,
1454
                              WirePointer::LIST, orphanArena);
1455
      origRef->listRef.setInlineComposite(totalSize);
1456

1457 1458 1459 1460
      WirePointer* newTag = reinterpret_cast<WirePointer*>(newPtr);
      newTag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, elementCount);
      newTag->structRef.set(newDataSize, newPointerCount);
      newPtr += POINTER_SIZE_IN_WORDS;
1461

1462 1463
      word* src = oldPtr;
      word* dst = newPtr;
1464
      for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
1465
        // Copy data section.
1466
        copyMemory(dst, src, oldDataSize);
1467

1468 1469 1470
        // Copy pointer section.
        WirePointer* newPointerSection = reinterpret_cast<WirePointer*>(dst + newDataSize);
        WirePointer* oldPointerSection = reinterpret_cast<WirePointer*>(src + oldDataSize);
1471
        for (auto j: kj::zeroTo(oldPointerCount)) {
1472
          transferPointer(origSegment, newPointerSection + j, oldSegment, oldPointerSection + j);
1473 1474
        }

1475 1476
        dst += newStep * (ONE * ELEMENTS);
        src += oldStep * (ONE * ELEMENTS);
1477
      }
1478

1479
      auto oldSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1480
            oldStep * upgradeBound<uint64_t>(elementCount),
1481 1482
            []() { KJ_FAIL_ASSERT("old size overflows but new size doesn't?"); });

1483
      // Zero out old location.  See explanation in getWritableStructPointer().
1484
      // Make sure to include the tag word.
1485
      zeroMemory(oldPtr - POINTER_SIZE_IN_WORDS, oldSize + POINTER_SIZE_IN_WORDS);
1486

1487
      return ListBuilder(origSegment, capTable, newPtr, newStep * BITS_PER_WORD, elementCount,
1488 1489
                         newDataSize * BITS_PER_WORD, newPointerCount,
                         ElementSize::INLINE_COMPOSITE);
1490
    } else {
1491
      // We're upgrading from a non-struct list.
1492

1493 1494
      auto oldDataSize = dataBitsPerElement(oldSize) * ELEMENTS;
      auto oldPointerCount = pointersPerElement(oldSize) * ELEMENTS;
1495
      auto oldStep = (oldDataSize + oldPointerCount * BITS_PER_POINTER) / ELEMENTS;
1496
      auto elementCount = oldRef->listRef.elementCount();
1497

1498
      if (oldSize == ElementSize::VOID) {
1499
        // Nothing to copy, just allocate a new list.
1500
        return initStructListPointer(origRef, origSegment, capTable, elementCount, elementSize);
1501
      } else {
1502
        // Upgrading to an inline composite list.
1503

1504
        KJ_REQUIRE(oldSize != ElementSize::BIT,
1505 1506 1507 1508 1509
            "Found bit list where struct list was expected; upgrading boolean lists to structs "
            "is no longer supported.") {
          goto useDefault;
        }

1510 1511
        auto newDataSize = elementSize.data;
        auto newPointerCount = elementSize.pointers;
1512

1513
        if (oldSize == ElementSize::POINTER) {
1514
          newPointerCount = kj::max(newPointerCount, ONE * POINTERS);
1515 1516
        } else {
          // Old list contains data elements, so we need at least 1 word of data.
1517
          newDataSize = kj::max(newDataSize, ONE * WORDS);
1518
        }
1519

1520
        auto newStep = (newDataSize + newPointerCount * WORDS_PER_POINTER) / ELEMENTS;
1521
        auto totalWords = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1522
              newStep * upgradeBound<uint64_t>(elementCount),
1523
              []() {KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size");});
1524

1525 1526
        // Don't let allocate() zero out the object just yet.
        zeroPointerAndFars(origSegment, origRef);
1527

1528
        word* newPtr = allocate(origRef, origSegment, capTable, totalWords + POINTER_SIZE_IN_WORDS,
1529
                                WirePointer::LIST, orphanArena);
1530
        origRef->listRef.setInlineComposite(totalWords);
1531

1532 1533 1534 1535
        WirePointer* tag = reinterpret_cast<WirePointer*>(newPtr);
        tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, elementCount);
        tag->structRef.set(newDataSize, newPointerCount);
        newPtr += POINTER_SIZE_IN_WORDS;
1536

1537
        if (oldSize == ElementSize::POINTER) {
1538 1539
          WirePointer* dst = reinterpret_cast<WirePointer*>(newPtr + newDataSize);
          WirePointer* src = reinterpret_cast<WirePointer*>(oldPtr);
1540
          for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
1541
            transferPointer(origSegment, dst, oldSegment, src);
1542
            dst += newStep / WORDS_PER_POINTER * (ONE * ELEMENTS);
1543
            ++src;
1544
          }
1545
        } else {
1546 1547 1548 1549 1550 1551 1552 1553
          byte* dst = reinterpret_cast<byte*>(newPtr);
          byte* src = reinterpret_cast<byte*>(oldPtr);
          auto newByteStep = newStep * (ONE * ELEMENTS) * BYTES_PER_WORD;
          auto oldByteStep = oldDataSize / BITS_PER_BYTE;
          for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
            copyMemory(dst, src, oldByteStep);
            src += oldByteStep;
            dst += newByteStep;
1554 1555
          }
        }
1556

1557
        auto oldSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1558
              roundBitsUpToWords(oldStep * upgradeBound<uint64_t>(elementCount)),
1559 1560
              []() { KJ_FAIL_ASSERT("old size overflows but new size doesn't?"); });

1561
        // Zero out old location.  See explanation in getWritableStructPointer().
1562
        zeroMemory(oldPtr, oldSize);
1563

1564
        return ListBuilder(origSegment, capTable, newPtr, newStep * BITS_PER_WORD, elementCount,
1565
                           newDataSize * BITS_PER_WORD, newPointerCount,
1566
                           ElementSize::INLINE_COMPOSITE);
1567
      }
1568 1569 1570
    }
  }

1571
  static KJ_ALWAYS_INLINE(SegmentAnd<Text::Builder> initTextPointer(
1572
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, TextSize size,
1573
      BuilderArena* orphanArena = nullptr)) {
Kenton Varda's avatar
Kenton Varda committed
1574
    // The byte list must include a NUL terminator.
1575
    auto byteSize = size + ONE * BYTES;
Kenton Varda's avatar
Kenton Varda committed
1576 1577

    // Allocate the space.
1578
    word* ptr = allocate(
1579
        ref, segment, capTable, roundBytesUpToWords(byteSize), WirePointer::LIST, orphanArena);
Kenton Varda's avatar
Kenton Varda committed
1580

1581
    // Initialize the pointer.
1582
    ref->listRef.set(ElementSize::BYTE, byteSize * (ONE * ELEMENTS / BYTES));
Kenton Varda's avatar
Kenton Varda committed
1583 1584

    // Build the Text::Builder.  This will initialize the NUL terminator.
1585
    return { segment, Text::Builder(reinterpret_cast<char*>(ptr), unbound(size / BYTES)) };
Kenton Varda's avatar
Kenton Varda committed
1586 1587
  }

1588
  static KJ_ALWAYS_INLINE(SegmentAnd<Text::Builder> setTextPointer(
1589
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, Text::Reader value,
1590
      BuilderArena* orphanArena = nullptr)) {
1591
    TextSize size = assertMax<MAX_TEXT_SIZE>(bounded(value.size()),
1592 1593 1594
        []() { KJ_FAIL_REQUIRE("text blob too big"); }) * BYTES;

    auto allocation = initTextPointer(ref, segment, capTable, size, orphanArena);
1595
    copyMemory(allocation.value.begin(), value);
1596
    return allocation;
Kenton Varda's avatar
Kenton Varda committed
1597 1598
  }

1599
  static KJ_ALWAYS_INLINE(Text::Builder getWritableTextPointer(
1600
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
1601 1602
      const void* defaultValue, TextSize defaultSize)) {
    return getWritableTextPointer(ref, ref->target(), segment,capTable,  defaultValue, defaultSize);
1603 1604 1605
  }

  static KJ_ALWAYS_INLINE(Text::Builder getWritableTextPointer(
1606
      WirePointer* ref, word* refTarget, SegmentBuilder* segment, CapTableBuilder* capTable,
1607
      const void* defaultValue, TextSize defaultSize)) {
Kenton Varda's avatar
Kenton Varda committed
1608
    if (ref->isNull()) {
1609
    useDefault:
1610
      if (defaultSize == ZERO * BYTES) {
1611 1612
        return nullptr;
      } else {
1613
        Text::Builder builder = initTextPointer(ref, segment, capTable, defaultSize).value;
1614 1615
        copyMemory(builder.asBytes().begin(), reinterpret_cast<const byte*>(defaultValue),
                   defaultSize);
1616 1617
        return builder;
      }
Kenton Varda's avatar
Kenton Varda committed
1618
    } else {
1619
      word* ptr = followFars(ref, refTarget, segment);
1620
      byte* bptr = reinterpret_cast<byte*>(ptr);
Kenton Varda's avatar
Kenton Varda committed
1621

1622
      KJ_REQUIRE(ref->kind() == WirePointer::LIST,
1623 1624 1625
          "Called getText{Field,Element}() but existing pointer is not a list.") {
        goto useDefault;
      }
1626
      KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
1627 1628 1629
          "Called getText{Field,Element}() but existing list pointer is not byte-sized.") {
        goto useDefault;
      }
Kenton Varda's avatar
Kenton Varda committed
1630

1631 1632 1633 1634 1635 1636
      auto maybeSize = trySubtract(ref->listRef.elementCount() * (ONE * BYTES / ELEMENTS),
                                   ONE * BYTES);
      KJ_IF_MAYBE(size, maybeSize) {
        KJ_REQUIRE(*(bptr + *size) == '\0', "Text blob missing NUL terminator.") {
          goto useDefault;
        }
1637

1638 1639 1640 1641 1642 1643
        return Text::Builder(reinterpret_cast<char*>(bptr), unbound(*size / BYTES));
      } else {
        KJ_FAIL_REQUIRE("zero-size blob can't be text (need NUL terminator)") {
          goto useDefault;
        };
      }
Kenton Varda's avatar
Kenton Varda committed
1644 1645 1646
    }
  }

1647
  static KJ_ALWAYS_INLINE(SegmentAnd<Data::Builder> initDataPointer(
1648
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, BlobSize size,
1649
      BuilderArena* orphanArena = nullptr)) {
Kenton Varda's avatar
Kenton Varda committed
1650
    // Allocate the space.
1651 1652
    word* ptr = allocate(ref, segment, capTable, roundBytesUpToWords(size),
                         WirePointer::LIST, orphanArena);
Kenton Varda's avatar
Kenton Varda committed
1653

1654
    // Initialize the pointer.
1655
    ref->listRef.set(ElementSize::BYTE, size * (ONE * ELEMENTS / BYTES));
Kenton Varda's avatar
Kenton Varda committed
1656 1657

    // Build the Data::Builder.
1658
    return { segment, Data::Builder(reinterpret_cast<byte*>(ptr), unbound(size / BYTES)) };
Kenton Varda's avatar
Kenton Varda committed
1659 1660
  }

1661
  static KJ_ALWAYS_INLINE(SegmentAnd<Data::Builder> setDataPointer(
1662
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, Data::Reader value,
1663
      BuilderArena* orphanArena = nullptr)) {
1664
    BlobSize size = assertMaxBits<BLOB_SIZE_BITS>(bounded(value.size()),
1665 1666 1667
        []() { KJ_FAIL_REQUIRE("text blob too big"); }) * BYTES;

    auto allocation = initDataPointer(ref, segment, capTable, size, orphanArena);
1668
    copyMemory(allocation.value.begin(), value);
1669
    return allocation;
Kenton Varda's avatar
Kenton Varda committed
1670 1671
  }

1672
  static KJ_ALWAYS_INLINE(Data::Builder getWritableDataPointer(
1673
      WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
1674
      const void* defaultValue, BlobSize defaultSize)) {
1675
    return getWritableDataPointer(ref, ref->target(), segment, capTable, defaultValue, defaultSize);
1676 1677 1678
  }

  static KJ_ALWAYS_INLINE(Data::Builder getWritableDataPointer(
1679
      WirePointer* ref, word* refTarget, SegmentBuilder* segment, CapTableBuilder* capTable,
1680
      const void* defaultValue, BlobSize defaultSize)) {
Kenton Varda's avatar
Kenton Varda committed
1681
    if (ref->isNull()) {
1682
    useDefault:
1683
      if (defaultSize == ZERO * BYTES) {
1684 1685
        return nullptr;
      } else {
1686
        Data::Builder builder = initDataPointer(ref, segment, capTable, defaultSize).value;
1687
        copyMemory(builder.begin(), reinterpret_cast<const byte*>(defaultValue), defaultSize);
1688 1689
        return builder;
      }
Kenton Varda's avatar
Kenton Varda committed
1690
    } else {
1691
      word* ptr = followFars(ref, refTarget, segment);
Kenton Varda's avatar
Kenton Varda committed
1692

1693
      KJ_REQUIRE(ref->kind() == WirePointer::LIST,
1694 1695 1696
          "Called getData{Field,Element}() but existing pointer is not a list.") {
        goto useDefault;
      }
1697
      KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
1698 1699 1700
          "Called getData{Field,Element}() but existing list pointer is not byte-sized.") {
        goto useDefault;
      }
Kenton Varda's avatar
Kenton Varda committed
1701

1702
      return Data::Builder(reinterpret_cast<byte*>(ptr),
1703
          unbound(ref->listRef.elementCount() / ELEMENTS));
Kenton Varda's avatar
Kenton Varda committed
1704 1705 1706
    }
  }

1707
  static SegmentAnd<word*> setStructPointer(
1708
      SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, StructReader value,
Matthew Maurer's avatar
Matthew Maurer committed
1709
      BuilderArena* orphanArena = nullptr, bool canonical = false) {
1710 1711
    auto dataSize = roundBitsUpToBytes(value.dataSize);
    auto ptrCount = value.pointerCount;
Matthew Maurer's avatar
Matthew Maurer committed
1712 1713

    if (canonical) {
1714
      // StructReaders should not have bitwidths other than 1, but let's be safe
1715 1716
      KJ_REQUIRE((value.dataSize == ONE * BITS)
                 || (value.dataSize % BITS_PER_BYTE == ZERO * BITS));
1717

1718 1719 1720 1721
      if (value.dataSize == ONE * BITS) {
        // Handle the truncation case where it's a false in a 1-bit struct
        if (!value.getDataField<bool>(ZERO * ELEMENTS)) {
          dataSize = ZERO * BYTES;
1722 1723 1724
        }
      } else {
        // Truncate the data section
1725 1726 1727 1728
        auto data = value.getDataSectionAsBlob();
        auto end = data.end();
        while (end > data.begin() && end[-1] == 0) --end;
        dataSize = intervalLength(data.begin(), end, MAX_STUCT_DATA_WORDS * BYTES_PER_WORD);
Matthew Maurer's avatar
Matthew Maurer committed
1729
      }
1730

Matthew Maurer's avatar
Matthew Maurer committed
1731
      // Truncate pointer section
1732 1733 1734
      const WirePointer* ptr = value.pointers + ptrCount;
      while (ptr > value.pointers && ptr[-1].isNull()) --ptr;
      ptrCount = intervalLength(value.pointers, ptr, MAX_STRUCT_POINTER_COUNT);
Matthew Maurer's avatar
Matthew Maurer committed
1735 1736
    }

1737
    auto dataWords = roundBytesUpToWords(dataSize);
1738

1739
    auto totalSize = dataWords + ptrCount * WORDS_PER_POINTER;
1740

1741
    word* ptr = allocate(ref, segment, capTable, totalSize, WirePointer::STRUCT, orphanArena);
1742
    ref->structRef.set(dataWords, ptrCount);
1743

1744
    if (value.dataSize == ONE * BITS) {
Matthew Maurer's avatar
Matthew Maurer committed
1745
      // Data size could be made 0 by truncation
1746 1747
      if (dataSize != ZERO * BYTES) {
        *reinterpret_cast<char*>(ptr) = value.getDataField<bool>(ZERO * ELEMENTS);
Matthew Maurer's avatar
Matthew Maurer committed
1748
      }
1749
    } else {
1750 1751 1752
      copyMemory(reinterpret_cast<byte*>(ptr),
                 reinterpret_cast<const byte*>(value.data),
                 dataSize);
1753 1754
    }

1755
    WirePointer* pointerSection = reinterpret_cast<WirePointer*>(ptr + dataWords);
1756
    for (auto i: kj::zeroTo(ptrCount)) {
1757
      copyPointer(segment, capTable, pointerSection + i,
Matthew Maurer's avatar
Matthew Maurer committed
1758 1759
                  value.segment, value.capTable, value.pointers + i,
                  value.nestingLimit, nullptr, canonical);
1760
    }
1761

1762
    return { segment, ptr };
1763 1764
  }

1765
#if !CAPNP_LITE
1766
  static void setCapabilityPointer(
1767 1768 1769 1770
      SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref,
      kj::Own<ClientHook>&& cap) {
    if (!ref->isNull()) {
      zeroObject(segment, capTable, ref);
1771
    }
1772
    if (cap->isNull()) {
1773
      zeroMemory(ref);
1774 1775 1776
    } else {
      ref->setCap(capTable->injectCap(kj::mv(cap)));
    }
1777
  }
1778
#endif  // !CAPNP_LITE
1779

1780
  static SegmentAnd<word*> setListPointer(
1781
      SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, ListReader value,
Matthew Maurer's avatar
Matthew Maurer committed
1782
      BuilderArena* orphanArena = nullptr, bool canonical = false) {
1783
    auto totalSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
1784
        roundBitsUpToWords(upgradeBound<uint64_t>(value.elementCount) * value.step),
1785
        []() { KJ_FAIL_ASSERT("encountered impossibly long struct list ListReader"); });
1786

1787
    if (value.elementSize != ElementSize::INLINE_COMPOSITE) {
1788
      // List of non-structs.
1789
      word* ptr = allocate(ref, segment, capTable, totalSize, WirePointer::LIST, orphanArena);
1790

1791
      if (value.elementSize == ElementSize::POINTER) {
1792
        // List of pointers.
1793
        ref->listRef.set(ElementSize::POINTER, value.elementCount);
1794
        for (auto i: kj::zeroTo(value.elementCount * (ONE * POINTERS / ELEMENTS))) {
1795 1796 1797
          copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(ptr) + i,
                      value.segment, value.capTable,
                      reinterpret_cast<const WirePointer*>(value.ptr) + i,
Matthew Maurer's avatar
Matthew Maurer committed
1798
                      value.nestingLimit, nullptr, canonical);
1799 1800 1801
        }
      } else {
        // List of data.
1802
        ref->listRef.set(value.elementSize, value.elementCount);
1803
        copyMemory(ptr, reinterpret_cast<const word*>(value.ptr), totalSize);
1804
      }
1805

1806
      return { segment, ptr };
1807 1808
    } else {
      // List of structs.
1809 1810
      StructDataWordCount declDataSize = value.structDataSize / BITS_PER_WORD;
      StructPointerCount declPointerCount = value.structPointerCount;
Matthew Maurer's avatar
Matthew Maurer committed
1811

1812 1813
      StructDataWordCount dataSize = ZERO * WORDS;
      StructPointerCount ptrCount = ZERO * POINTERS;
Matthew Maurer's avatar
Matthew Maurer committed
1814 1815

      if (canonical) {
1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830
        for (auto i: kj::zeroTo(value.elementCount)) {
          auto element = value.getStructElement(i);

          // Truncate the data section
          auto data = element.getDataSectionAsBlob();
          auto end = data.end();
          while (end > data.begin() && end[-1] == 0) --end;
          dataSize = kj::max(dataSize, roundBytesUpToWords(
              intervalLength(data.begin(), end, MAX_STUCT_DATA_WORDS * BYTES_PER_WORD)));

          // Truncate pointer section
          const WirePointer* ptr = element.pointers + element.pointerCount;
          while (ptr > element.pointers && ptr[-1].isNull()) --ptr;
          ptrCount = kj::max(ptrCount,
              intervalLength(element.pointers, ptr, MAX_STRUCT_POINTER_COUNT));
Matthew Maurer's avatar
Matthew Maurer committed
1831
        }
1832
        auto newTotalSize = (dataSize + upgradeBound<uint64_t>(ptrCount) * WORDS_PER_POINTER)
1833 1834
            / ELEMENTS * value.elementCount;
        KJ_ASSERT(newTotalSize <= totalSize);  // we've only removed data!
1835
        totalSize = assumeMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(newTotalSize);
Matthew Maurer's avatar
Matthew Maurer committed
1836 1837 1838 1839 1840
      } else {
        dataSize = declDataSize;
        ptrCount = declPointerCount;
      }

1841
      KJ_DASSERT(value.structDataSize % BITS_PER_WORD == ZERO * BITS);
1842
      word* ptr = allocate(ref, segment, capTable, totalSize + POINTER_SIZE_IN_WORDS,
1843
                           WirePointer::LIST, orphanArena);
1844 1845 1846 1847
      ref->listRef.setInlineComposite(totalSize);

      WirePointer* tag = reinterpret_cast<WirePointer*>(ptr);
      tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, value.elementCount);
Matthew Maurer's avatar
Matthew Maurer committed
1848
      tag->structRef.set(dataSize, ptrCount);
1849
      word* dst = ptr + POINTER_SIZE_IN_WORDS;
1850 1851

      const word* src = reinterpret_cast<const word*>(value.ptr);
1852 1853
      for (auto i KJ_UNUSED: kj::zeroTo(value.elementCount)) {
        copyMemory(dst, src, dataSize);
1854
        dst += dataSize;
Matthew Maurer's avatar
Matthew Maurer committed
1855
        src += declDataSize;
1856

1857 1858 1859
        for (auto j: kj::zeroTo(ptrCount)) {
          copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(dst) + j,
              value.segment, value.capTable, reinterpret_cast<const WirePointer*>(src) + j,
Matthew Maurer's avatar
Matthew Maurer committed
1860
              value.nestingLimit, nullptr, canonical);
1861
        }
1862 1863
        dst += ptrCount * WORDS_PER_POINTER;
        src += declPointerCount * WORDS_PER_POINTER;
1864
      }
1865

1866
      return { segment, ptr };
1867 1868 1869
    }
  }

1870
  static KJ_ALWAYS_INLINE(SegmentAnd<word*> copyPointer(
1871 1872
      SegmentBuilder* dstSegment, CapTableBuilder* dstCapTable, WirePointer* dst,
      SegmentReader* srcSegment, CapTableReader* srcCapTable, const WirePointer* src,
Matthew Maurer's avatar
Matthew Maurer committed
1873 1874
      int nestingLimit, BuilderArena* orphanArena = nullptr,
      bool canonical = false)) {
1875 1876
    return copyPointer(dstSegment, dstCapTable, dst,
                       srcSegment, srcCapTable, src, src->target(),
Matthew Maurer's avatar
Matthew Maurer committed
1877
                       nestingLimit, orphanArena, canonical);
1878 1879 1880
  }

  static SegmentAnd<word*> copyPointer(
1881 1882
      SegmentBuilder* dstSegment, CapTableBuilder* dstCapTable, WirePointer* dst,
      SegmentReader* srcSegment, CapTableReader* srcCapTable, const WirePointer* src,
Matthew Maurer's avatar
Matthew Maurer committed
1883 1884
      const word* srcTarget, int nestingLimit,
      BuilderArena* orphanArena = nullptr, bool canonical = false) {
1885 1886 1887 1888
    // Deep-copy the object pointed to by src into dst.  It turns out we can't reuse
    // readStructPointer(), etc. because they do type checking whereas here we want to accept any
    // valid pointer.

1889
    if (src->isNull()) {
1890
    useDefault:
1891
      if (!dst->isNull()) {
1892
        zeroObject(dstSegment, dstCapTable, dst);
1893
        zeroMemory(dst);
1894
      }
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906
      return { dstSegment, nullptr };
    }

    const word* ptr = WireHelpers::followFars(src, srcTarget, srcSegment);
    if (KJ_UNLIKELY(ptr == nullptr)) {
      // Already reported the error.
      goto useDefault;
    }

    switch (src->kind()) {
      case WirePointer::STRUCT:
        KJ_REQUIRE(nestingLimit > 0,
David Renshaw's avatar
David Renshaw committed
1907
              "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
1908 1909 1910 1911 1912 1913 1914
          goto useDefault;
        }

        KJ_REQUIRE(boundsCheck(srcSegment, ptr, ptr + src->structRef.wordSize()),
                   "Message contained out-of-bounds struct pointer.") {
          goto useDefault;
        }
1915 1916
        return setStructPointer(dstSegment, dstCapTable, dst,
            StructReader(srcSegment, srcCapTable, ptr,
1917 1918 1919
                         reinterpret_cast<const WirePointer*>(ptr + src->structRef.dataSize.get()),
                         src->structRef.dataSize.get() * BITS_PER_WORD,
                         src->structRef.ptrCount.get(),
1920
                         nestingLimit - 1),
Matthew Maurer's avatar
Matthew Maurer committed
1921
            orphanArena, canonical);
1922 1923

      case WirePointer::LIST: {
1924
        ElementSize elementSize = src->listRef.elementSize();
1925 1926

        KJ_REQUIRE(nestingLimit > 0,
David Renshaw's avatar
David Renshaw committed
1927
              "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
1928 1929 1930
          goto useDefault;
        }

1931
        if (elementSize == ElementSize::INLINE_COMPOSITE) {
1932
          auto wordCount = src->listRef.inlineCompositeWordCount();
1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945
          const WirePointer* tag = reinterpret_cast<const WirePointer*>(ptr);
          ptr += POINTER_SIZE_IN_WORDS;

          KJ_REQUIRE(boundsCheck(srcSegment, ptr - POINTER_SIZE_IN_WORDS, ptr + wordCount),
                     "Message contains out-of-bounds list pointer.") {
            goto useDefault;
          }

          KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
                     "INLINE_COMPOSITE lists of non-STRUCT type are not supported.") {
            goto useDefault;
          }

1946
          auto elementCount = tag->inlineCompositeListElementCount();
1947 1948
          auto wordsPerElement = tag->structRef.wordSize() / ELEMENTS;

1949
          KJ_REQUIRE(wordsPerElement * upgradeBound<uint64_t>(elementCount) <= wordCount,
1950 1951 1952 1953
                     "INLINE_COMPOSITE list's elements overrun its word count.") {
            goto useDefault;
          }

1954
          if (wordsPerElement * (ONE * ELEMENTS) == ZERO * WORDS) {
1955 1956
            // Watch out for lists of zero-sized structs, which can claim to be arbitrarily large
            // without having sent actual data.
1957
            KJ_REQUIRE(amplifiedRead(srcSegment, elementCount * (ONE * WORDS / ELEMENTS)),
1958 1959 1960 1961 1962
                       "Message contains amplified list pointer.") {
              goto useDefault;
            }
          }

1963 1964 1965
          return setListPointer(dstSegment, dstCapTable, dst,
              ListReader(srcSegment, srcCapTable, ptr,
                         elementCount, wordsPerElement * BITS_PER_WORD,
1966
                         tag->structRef.dataSize.get() * BITS_PER_WORD,
1967
                         tag->structRef.ptrCount.get(), ElementSize::INLINE_COMPOSITE,
1968
                         nestingLimit - 1),
Matthew Maurer's avatar
Matthew Maurer committed
1969
              orphanArena, canonical);
1970
        } else {
1971 1972
          auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
          auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
1973
          auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
1974
          auto elementCount = src->listRef.elementCount();
1975
          auto wordCount = roundBitsUpToWords(upgradeBound<uint64_t>(elementCount) * step);
1976 1977 1978 1979 1980 1981

          KJ_REQUIRE(boundsCheck(srcSegment, ptr, ptr + wordCount),
                     "Message contains out-of-bounds list pointer.") {
            goto useDefault;
          }

1982 1983 1984
          if (elementSize == ElementSize::VOID) {
            // Watch out for lists of void, which can claim to be arbitrarily large without having
            // sent actual data.
1985
            KJ_REQUIRE(amplifiedRead(srcSegment, elementCount * (ONE * WORDS / ELEMENTS)),
1986 1987 1988 1989 1990
                       "Message contains amplified list pointer.") {
              goto useDefault;
            }
          }

1991 1992 1993
          return setListPointer(dstSegment, dstCapTable, dst,
              ListReader(srcSegment, srcCapTable, ptr, elementCount, step, dataSize, pointerCount,
                         elementSize, nestingLimit - 1),
Matthew Maurer's avatar
Matthew Maurer committed
1994
              orphanArena, canonical);
1995 1996 1997
        }
      }

1998
      case WirePointer::FAR:
1999
        KJ_FAIL_REQUIRE("Unexpected FAR pointer.") {
2000 2001 2002
          goto useDefault;
        }

2003 2004
      case WirePointer::OTHER: {
        KJ_REQUIRE(src->isCapability(), "Unknown pointer type.") {
2005 2006 2007
          goto useDefault;
        }

Matthew Maurer's avatar
Matthew Maurer committed
2008 2009
        if (canonical) {
          KJ_FAIL_REQUIRE("Cannot create a canonical message with a capability") {
2010
            break;
Matthew Maurer's avatar
Matthew Maurer committed
2011 2012
          }
        }
2013
#if !CAPNP_LITE
2014 2015
        KJ_IF_MAYBE(cap, srcCapTable->extractCap(src->capRef.index.get())) {
          setCapabilityPointer(dstSegment, dstCapTable, dst, kj::mv(*cap));
2016 2017
          // Return dummy non-null pointer so OrphanBuilder doesn't end up null.
          return { dstSegment, reinterpret_cast<word*>(1) };
2018
        } else {
2019
#endif  // !CAPNP_LITE
2020
          KJ_FAIL_REQUIRE("Message contained invalid capability pointer.") {
2021 2022
            goto useDefault;
          }
2023
#if !CAPNP_LITE
2024
        }
2025
#endif  // !CAPNP_LITE
2026
      }
2027
    }
Kenton Varda's avatar
Kenton Varda committed
2028 2029

    KJ_UNREACHABLE;
2030 2031
  }

2032 2033
  static void adopt(SegmentBuilder* segment, CapTableBuilder* capTable,
                    WirePointer* ref, OrphanBuilder&& value) {
2034
    KJ_REQUIRE(value.segment == nullptr || value.segment->getArena() == segment->getArena(),
2035 2036 2037
               "Adopted object must live in the same message.");

    if (!ref->isNull()) {
2038
      zeroObject(segment, capTable, ref);
2039 2040
    }

2041
    if (value == nullptr) {
2042
      // Set null.
2043
      zeroMemory(ref);
2044
    } else if (value.tagAsPtr()->isPositional()) {
2045
      WireHelpers::transferPointer(segment, ref, value.segment, value.tagAsPtr(), value.location);
2046 2047
    } else {
      // FAR and OTHER pointers are position-independent, so we can just copy.
2048
      copyMemory(ref, value.tagAsPtr());
2049 2050 2051
    }

    // Take ownership away from the OrphanBuilder.
2052
    zeroMemory(value.tagAsPtr());
2053 2054 2055 2056
    value.location = nullptr;
    value.segment = nullptr;
  }

2057 2058
  static OrphanBuilder disown(SegmentBuilder* segment, CapTableBuilder* capTable,
                              WirePointer* ref) {
2059 2060 2061 2062
    word* location;

    if (ref->isNull()) {
      location = nullptr;
2063 2064
    } else if (ref->kind() == WirePointer::OTHER) {
      KJ_REQUIRE(ref->isCapability(), "Unknown pointer type.") { break; }
2065
      location = reinterpret_cast<word*>(1);  // dummy so that it is non-null
2066 2067
    } else {
      WirePointer* refCopy = ref;
2068
      location = followFarsNoWritableCheck(refCopy, ref->target(), segment);
2069 2070
    }

2071
    OrphanBuilder result(ref, segment, capTable, location);
2072

2073
    if (!ref->isNull() && ref->isPositional()) {
2074
      result.tagAsPtr()->setKindForOrphan(ref->kind());
2075
    }
2076 2077

    // Zero out the pointer that was disowned.
2078
    zeroMemory(ref);
2079 2080

    return result;
2081 2082
  }

2083 2084
  // -----------------------------------------------------------------

2085
  static KJ_ALWAYS_INLINE(StructReader readStructPointer(
2086 2087
      SegmentReader* segment, CapTableReader* capTable,
      const WirePointer* ref, const word* defaultValue,
2088
      int nestingLimit)) {
2089
    return readStructPointer(segment, capTable, ref, ref->target(), defaultValue, nestingLimit);
2090 2091 2092
  }

  static KJ_ALWAYS_INLINE(StructReader readStructPointer(
2093 2094
      SegmentReader* segment, CapTableReader* capTable,
      const WirePointer* ref, const word* refTarget,
2095
      const word* defaultValue, int nestingLimit)) {
2096
    if (ref->isNull()) {
2097
    useDefault:
Kenton Varda's avatar
Kenton Varda committed
2098
      if (defaultValue == nullptr ||
2099
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
2100
        return StructReader();
2101
      }
2102
      segment = nullptr;
2103
      ref = reinterpret_cast<const WirePointer*>(defaultValue);
2104
      refTarget = ref->target();
2105 2106
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }
2107

2108
    KJ_REQUIRE(nestingLimit > 0,
David Renshaw's avatar
David Renshaw committed
2109
               "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
2110 2111
      goto useDefault;
    }
2112

2113
    const word* ptr = followFars(ref, refTarget, segment);
2114
    if (KJ_UNLIKELY(ptr == nullptr)) {
2115 2116 2117
      // Already reported the error.
      goto useDefault;
    }
2118

2119 2120
    KJ_REQUIRE(ref->kind() == WirePointer::STRUCT,
               "Message contains non-struct pointer where struct pointer was expected.") {
2121 2122 2123
      goto useDefault;
    }

2124 2125
    KJ_REQUIRE(boundsCheck(segment, ptr, ptr + ref->structRef.wordSize()),
               "Message contained out-of-bounds struct pointer.") {
2126
      goto useDefault;
2127
    }
2128

2129
    return StructReader(
2130 2131
        segment, capTable,
        ptr, reinterpret_cast<const WirePointer*>(ptr + ref->structRef.dataSize.get()),
2132
        ref->structRef.dataSize.get() * BITS_PER_WORD,
2133
        ref->structRef.ptrCount.get(),
2134
        nestingLimit - 1);
2135 2136
  }

2137
#if !CAPNP_LITE
2138
  static KJ_ALWAYS_INLINE(kj::Own<ClientHook> readCapabilityPointer(
2139 2140
      SegmentReader* segment, CapTableReader* capTable,
      const WirePointer* ref, int nestingLimit)) {
2141 2142
    kj::Maybe<kj::Own<ClientHook>> maybeCap;

2143 2144 2145 2146 2147 2148
    KJ_REQUIRE(brokenCapFactory != nullptr,
               "Trying to read capabilities without ever having created a capability context.  "
               "To read capabilities from a message, you must imbue it with CapReaderContext, or "
               "use the Cap'n Proto RPC system.");

    if (ref->isNull()) {
2149
      return brokenCapFactory->newNullCap();
2150 2151 2152 2153 2154
    } else if (!ref->isCapability()) {
      KJ_FAIL_REQUIRE(
          "Message contains non-capability pointer where capability pointer was expected.") {
        break;
      }
2155
      return brokenCapFactory->newBrokenCap(
2156
          "Calling capability extracted from a non-capability pointer.");
2157
    } else KJ_IF_MAYBE(cap, capTable->extractCap(ref->capRef.index.get())) {
2158 2159
      return kj::mv(*cap);
    } else {
2160 2161 2162 2163
      KJ_FAIL_REQUIRE("Message contains invalid capability pointer.") {
        break;
      }
      return brokenCapFactory->newBrokenCap("Calling invalid capability pointer.");
2164 2165
    }
  }
2166
#endif  // !CAPNP_LITE
2167

2168
  static KJ_ALWAYS_INLINE(ListReader readListPointer(
2169 2170
      SegmentReader* segment, CapTableReader* capTable,
      const WirePointer* ref, const word* defaultValue,
2171
      ElementSize expectedElementSize, int nestingLimit, bool checkElementSize = true)) {
2172
    return readListPointer(segment, capTable, ref, ref->target(), defaultValue,
2173
                           expectedElementSize, nestingLimit, checkElementSize);
2174 2175 2176
  }

  static KJ_ALWAYS_INLINE(ListReader readListPointer(
2177 2178
      SegmentReader* segment, CapTableReader* capTable,
      const WirePointer* ref, const word* refTarget,
2179
      const word* defaultValue, ElementSize expectedElementSize, int nestingLimit,
2180
      bool checkElementSize = true)) {
2181
    if (ref->isNull()) {
2182
    useDefault:
Kenton Varda's avatar
Kenton Varda committed
2183
      if (defaultValue == nullptr ||
2184
          reinterpret_cast<const WirePointer*>(defaultValue)->isNull()) {
2185
        return ListReader(expectedElementSize);
2186
      }
2187
      segment = nullptr;
2188
      ref = reinterpret_cast<const WirePointer*>(defaultValue);
2189
      refTarget = ref->target();
2190 2191
      defaultValue = nullptr;  // If the default value is itself invalid, don't use it again.
    }
2192

2193
    KJ_REQUIRE(nestingLimit > 0,
David Renshaw's avatar
David Renshaw committed
2194
               "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
2195 2196
      goto useDefault;
    }
2197

2198
    const word* ptr = followFars(ref, refTarget, segment);
2199
    if (KJ_UNLIKELY(ptr == nullptr)) {
2200 2201 2202 2203
      // Already reported error.
      goto useDefault;
    }

2204 2205
    KJ_REQUIRE(ref->kind() == WirePointer::LIST,
               "Message contains non-list pointer where list pointer was expected.") {
2206
      goto useDefault;
2207 2208
    }

2209 2210
    ElementSize elementSize = ref->listRef.elementSize();
    if (elementSize == ElementSize::INLINE_COMPOSITE) {
2211
      auto wordCount = ref->listRef.inlineCompositeWordCount();
2212

2213 2214 2215
      // An INLINE_COMPOSITE list points to a tag, which is formatted like a pointer.
      const WirePointer* tag = reinterpret_cast<const WirePointer*>(ptr);
      ptr += POINTER_SIZE_IN_WORDS;
2216

2217 2218
      KJ_REQUIRE(boundsCheck(segment, ptr - POINTER_SIZE_IN_WORDS, ptr + wordCount),
                 "Message contains out-of-bounds list pointer.") {
2219 2220
        goto useDefault;
      }
2221

2222 2223
      KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
                 "INLINE_COMPOSITE lists of non-STRUCT type are not supported.") {
2224 2225
        goto useDefault;
      }
2226

2227 2228
      auto size = tag->inlineCompositeListElementCount();
      auto wordsPerElement = tag->structRef.wordSize() / ELEMENTS;
2229

2230
      KJ_REQUIRE(upgradeBound<uint64_t>(size) * wordsPerElement <= wordCount,
2231
                 "INLINE_COMPOSITE list's elements overrun its word count.") {
2232 2233
        goto useDefault;
      }
2234

2235
      if (wordsPerElement * (ONE * ELEMENTS) == ZERO * WORDS) {
2236 2237
        // Watch out for lists of zero-sized structs, which can claim to be arbitrarily large
        // without having sent actual data.
2238
        KJ_REQUIRE(amplifiedRead(segment, size * (ONE * WORDS / ELEMENTS)),
2239 2240 2241 2242 2243
                   "Message contains amplified list pointer.") {
          goto useDefault;
        }
      }

2244 2245
      if (checkElementSize) {
        // If a struct list was not expected, then presumably a non-struct list was upgraded to a
2246 2247 2248
        // struct list. We need to manipulate the pointer to point at the first field of the
        // struct. Together with the `step` field, this will allow the struct list to be accessed
        // as if it were a primitive list without branching.
2249

2250 2251
        // Check whether the size is compatible.
        switch (expectedElementSize) {
2252
          case ElementSize::VOID:
2253
            break;
2254

2255
          case ElementSize::BIT:
2256 2257 2258 2259 2260 2261
            KJ_FAIL_REQUIRE(
                "Found struct list where bit list was expected; upgrading boolean lists to structs "
                "is no longer supported.") {
              goto useDefault;
            }
            break;
2262

2263 2264 2265 2266
          case ElementSize::BYTE:
          case ElementSize::TWO_BYTES:
          case ElementSize::FOUR_BYTES:
          case ElementSize::EIGHT_BYTES:
2267
            KJ_REQUIRE(tag->structRef.dataSize.get() > ZERO * WORDS,
2268 2269 2270 2271
                       "Expected a primitive list, but got a list of pointer-only structs.") {
              goto useDefault;
            }
            break;
2272

2273
          case ElementSize::POINTER:
2274 2275 2276 2277
            // We expected a list of pointers but got a list of structs.  Assuming the first field
            // in the struct is the pointer we were looking for, we want to munge the pointer to
            // point at the first element's pointer section.
            ptr += tag->structRef.dataSize.get();
2278
            KJ_REQUIRE(tag->structRef.ptrCount.get() > ZERO * POINTERS,
2279 2280 2281 2282
                       "Expected a pointer list, but got a list of data-only structs.") {
              goto useDefault;
            }
            break;
2283

2284
          case ElementSize::INLINE_COMPOSITE:
2285 2286
            break;
        }
2287 2288
      }

2289
      return ListReader(
2290
          segment, capTable, ptr, size, wordsPerElement * BITS_PER_WORD,
2291
          tag->structRef.dataSize.get() * BITS_PER_WORD,
2292
          tag->structRef.ptrCount.get(), ElementSize::INLINE_COMPOSITE,
2293
          nestingLimit - 1);
2294 2295

    } else {
2296
      // This is a primitive or pointer list, but all such lists can also be interpreted as struct
2297
      // lists.  We need to compute the data size and pointer count for such structs.
2298 2299 2300
      auto dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
      auto pointerCount = pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
      auto elementCount = ref->listRef.elementCount();
2301
      auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
2302

2303
      auto wordCount = roundBitsUpToWords(upgradeBound<uint64_t>(elementCount) * step);
2304
      KJ_REQUIRE(boundsCheck(segment, ptr, ptr + wordCount),
2305
            "Message contains out-of-bounds list pointer.") {
2306
        goto useDefault;
2307 2308
      }

2309 2310 2311
      if (elementSize == ElementSize::VOID) {
        // Watch out for lists of void, which can claim to be arbitrarily large without having sent
        // actual data.
2312
        KJ_REQUIRE(amplifiedRead(segment, elementCount * (ONE * WORDS / ELEMENTS)),
2313 2314 2315 2316 2317
                   "Message contains amplified list pointer.") {
          goto useDefault;
        }
      }

2318
      if (checkElementSize) {
2319
        if (elementSize == ElementSize::BIT && expectedElementSize != ElementSize::BIT) {
2320 2321 2322 2323 2324
          KJ_FAIL_REQUIRE(
              "Found bit list where struct list was expected; upgrading boolean lists to structs "
              "is no longer supported.") {
            goto useDefault;
          }
2325 2326
        }

2327 2328 2329 2330
        // Verify that the elements are at least as large as the expected type.  Note that if we
        // expected INLINE_COMPOSITE, the expected sizes here will be zero, because bounds checking
        // will be performed at field access time.  So this check here is for the case where we
        // expected a list of some primitive or pointer type.
2331

2332 2333 2334 2335
        BitCount expectedDataBitsPerElement =
            dataBitsPerElement(expectedElementSize) * ELEMENTS;
        WirePointerCount expectedPointersPerElement =
            pointersPerElement(expectedElementSize) * ELEMENTS;
2336

2337 2338 2339 2340 2341 2342 2343 2344
        KJ_REQUIRE(expectedDataBitsPerElement <= dataSize,
                   "Message contained list with incompatible element type.") {
          goto useDefault;
        }
        KJ_REQUIRE(expectedPointersPerElement <= pointerCount,
                   "Message contained list with incompatible element type.") {
          goto useDefault;
        }
2345
      }
2346

2347
      return ListReader(segment, capTable, ptr, elementCount, step,
2348
                        dataSize, pointerCount, elementSize, nestingLimit - 1);
2349 2350
    }
  }
Kenton Varda's avatar
Kenton Varda committed
2351

2352
  static KJ_ALWAYS_INLINE(Text::Reader readTextPointer(
2353
      SegmentReader* segment, const WirePointer* ref,
Kenton Varda's avatar
Kenton Varda committed
2354
      const void* defaultValue, ByteCount defaultSize)) {
2355 2356 2357 2358 2359 2360
    return readTextPointer(segment, ref, ref->target(), defaultValue, defaultSize);
  }

  static KJ_ALWAYS_INLINE(Text::Reader readTextPointer(
      SegmentReader* segment, const WirePointer* ref, const word* refTarget,
      const void* defaultValue, ByteCount defaultSize)) {
2361
    if (ref->isNull()) {
Kenton Varda's avatar
Kenton Varda committed
2362
    useDefault:
2363
      if (defaultValue == nullptr) defaultValue = "";
2364
      return Text::Reader(reinterpret_cast<const char*>(defaultValue),
2365
          unbound(defaultSize / BYTES));
Kenton Varda's avatar
Kenton Varda committed
2366
    } else {
2367
      const word* ptr = followFars(ref, refTarget, segment);
Kenton Varda's avatar
Kenton Varda committed
2368

2369
      if (KJ_UNLIKELY(ptr == nullptr)) {
2370
        // Already reported error.
Kenton Varda's avatar
Kenton Varda committed
2371 2372 2373
        goto useDefault;
      }

2374
      auto size = ref->listRef.elementCount() * (ONE * BYTES / ELEMENTS);
2375

2376 2377
      KJ_REQUIRE(ref->kind() == WirePointer::LIST,
                 "Message contains non-list pointer where text was expected.") {
Kenton Varda's avatar
Kenton Varda committed
2378 2379 2380
        goto useDefault;
      }

2381
      KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
2382
                 "Message contains list pointer of non-bytes where text was expected.") {
Kenton Varda's avatar
Kenton Varda committed
2383 2384 2385
        goto useDefault;
      }

2386
      KJ_REQUIRE(boundsCheck(segment, ptr, ptr + roundBytesUpToWords(size)),
2387
                 "Message contained out-of-bounds text pointer.") {
Kenton Varda's avatar
Kenton Varda committed
2388 2389 2390
        goto useDefault;
      }

2391
      KJ_REQUIRE(size > ZERO * BYTES, "Message contains text that is not NUL-terminated.") {
2392 2393 2394
        goto useDefault;
      }

Kenton Varda's avatar
Kenton Varda committed
2395
      const char* cptr = reinterpret_cast<const char*>(ptr);
2396
      uint unboundedSize = unbound(size / BYTES) - 1;
Kenton Varda's avatar
Kenton Varda committed
2397

2398
      KJ_REQUIRE(cptr[unboundedSize] == '\0', "Message contains text that is not NUL-terminated.") {
Kenton Varda's avatar
Kenton Varda committed
2399 2400 2401
        goto useDefault;
      }

2402
      return Text::Reader(cptr, unboundedSize);
Kenton Varda's avatar
Kenton Varda committed
2403 2404 2405
    }
  }

2406
  static KJ_ALWAYS_INLINE(Data::Reader readDataPointer(
2407
      SegmentReader* segment, const WirePointer* ref,
2408
      const void* defaultValue, BlobSize defaultSize)) {
2409 2410 2411 2412 2413
    return readDataPointer(segment, ref, ref->target(), defaultValue, defaultSize);
  }

  static KJ_ALWAYS_INLINE(Data::Reader readDataPointer(
      SegmentReader* segment, const WirePointer* ref, const word* refTarget,
2414
      const void* defaultValue, BlobSize defaultSize)) {
2415
    if (ref->isNull()) {
Kenton Varda's avatar
Kenton Varda committed
2416
    useDefault:
2417
      return Data::Reader(reinterpret_cast<const byte*>(defaultValue),
2418
          unbound(defaultSize / BYTES));
Kenton Varda's avatar
Kenton Varda committed
2419
    } else {
2420
      const word* ptr = followFars(ref, refTarget, segment);
Kenton Varda's avatar
Kenton Varda committed
2421

2422
      if (KJ_UNLIKELY(ptr == nullptr)) {
2423
        // Already reported error.
Kenton Varda's avatar
Kenton Varda committed
2424 2425 2426
        goto useDefault;
      }

2427
      auto size = ref->listRef.elementCount() * (ONE * BYTES / ELEMENTS);
2428

2429 2430
      KJ_REQUIRE(ref->kind() == WirePointer::LIST,
                 "Message contains non-list pointer where data was expected.") {
Kenton Varda's avatar
Kenton Varda committed
2431 2432 2433
        goto useDefault;
      }

2434
      KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
2435
                 "Message contains list pointer of non-bytes where data was expected.") {
Kenton Varda's avatar
Kenton Varda committed
2436 2437 2438
        goto useDefault;
      }

2439
      KJ_REQUIRE(boundsCheck(segment, ptr, ptr + roundBytesUpToWords(size)),
2440
                 "Message contained out-of-bounds data pointer.") {
Kenton Varda's avatar
Kenton Varda committed
2441 2442 2443
        goto useDefault;
      }

2444
      return Data::Reader(reinterpret_cast<const byte*>(ptr), unbound(size / BYTES));
Kenton Varda's avatar
Kenton Varda committed
2445 2446
    }
  }
2447 2448 2449
};

// =======================================================================================
2450
// PointerBuilder
2451

2452
StructBuilder PointerBuilder::initStruct(StructSize size) {
2453
  return WireHelpers::initStructPointer(pointer, segment, capTable, size);
2454 2455
}

2456
StructBuilder PointerBuilder::getStruct(StructSize size, const word* defaultValue) {
2457
  return WireHelpers::getWritableStructPointer(pointer, segment, capTable, size, defaultValue);
2458 2459
}

2460
ListBuilder PointerBuilder::initList(ElementSize elementSize, ElementCount elementCount) {
2461
  return WireHelpers::initListPointer(pointer, segment, capTable, elementCount, elementSize);
2462 2463
}

2464
ListBuilder PointerBuilder::initStructList(ElementCount elementCount, StructSize elementSize) {
2465
  return WireHelpers::initStructListPointer(pointer, segment, capTable, elementCount, elementSize);
2466 2467
}

2468
ListBuilder PointerBuilder::getList(ElementSize elementSize, const word* defaultValue) {
2469
  return WireHelpers::getWritableListPointer(pointer, segment, capTable, elementSize, defaultValue);
2470
}
2471

2472
ListBuilder PointerBuilder::getStructList(StructSize elementSize, const word* defaultValue) {
2473 2474
  return WireHelpers::getWritableStructListPointer(
      pointer, segment, capTable, elementSize, defaultValue);
2475 2476
}

2477
ListBuilder PointerBuilder::getListAnySize(const word* defaultValue) {
2478
  return WireHelpers::getWritableListPointerAnySize(pointer, segment, capTable, defaultValue);
2479 2480
}

2481 2482
template <>
Text::Builder PointerBuilder::initBlob<Text>(ByteCount size) {
2483 2484
  return WireHelpers::initTextPointer(pointer, segment, capTable,
      assertMax<MAX_TEXT_SIZE>(size, ThrowOverflow())).value;
2485 2486
}
template <>
2487
void PointerBuilder::setBlob<Text>(Text::Reader value) {
2488
  WireHelpers::setTextPointer(pointer, segment, capTable, value);
2489 2490 2491
}
template <>
Text::Builder PointerBuilder::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) {
2492 2493
  return WireHelpers::getWritableTextPointer(pointer, segment, capTable, defaultValue,
      assertMax<MAX_TEXT_SIZE>(defaultSize, ThrowOverflow()));
2494 2495
}

2496 2497
template <>
Data::Builder PointerBuilder::initBlob<Data>(ByteCount size) {
2498 2499
  return WireHelpers::initDataPointer(pointer, segment, capTable,
      assertMaxBits<BLOB_SIZE_BITS>(size, ThrowOverflow())).value;
2500 2501
}
template <>
2502
void PointerBuilder::setBlob<Data>(Data::Reader value) {
2503
  WireHelpers::setDataPointer(pointer, segment, capTable, value);
2504 2505 2506
}
template <>
Data::Builder PointerBuilder::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) {
2507 2508
  return WireHelpers::getWritableDataPointer(pointer, segment, capTable, defaultValue,
      assertMaxBits<BLOB_SIZE_BITS>(defaultSize, ThrowOverflow()));
2509 2510
}

Matthew Maurer's avatar
Matthew Maurer committed
2511 2512
void PointerBuilder::setStruct(const StructReader& value, bool canonical) {
  WireHelpers::setStructPointer(segment, capTable, pointer, value, nullptr, canonical);
2513 2514
}

Matthew Maurer's avatar
Matthew Maurer committed
2515 2516
void PointerBuilder::setList(const ListReader& value, bool canonical) {
  WireHelpers::setListPointer(segment, capTable, pointer, value, nullptr, canonical);
2517 2518
}

2519
#if !CAPNP_LITE
2520
kj::Own<ClientHook> PointerBuilder::getCapability() {
2521
  return WireHelpers::readCapabilityPointer(
2522
      segment, capTable, pointer, kj::maxValue);
2523 2524
}

2525
void PointerBuilder::setCapability(kj::Own<ClientHook>&& cap) {
2526
  WireHelpers::setCapabilityPointer(segment, capTable, pointer, kj::mv(cap));
2527
}
2528
#endif  // !CAPNP_LITE
2529

2530
void PointerBuilder::adopt(OrphanBuilder&& value) {
2531
  WireHelpers::adopt(segment, capTable, pointer, kj::mv(value));
Kenton Varda's avatar
Kenton Varda committed
2532
}
2533 2534

OrphanBuilder PointerBuilder::disown() {
2535
  return WireHelpers::disown(segment, capTable, pointer);
Kenton Varda's avatar
Kenton Varda committed
2536 2537
}

2538
void PointerBuilder::clear() {
2539
  WireHelpers::zeroObject(segment, capTable, pointer);
2540
  WireHelpers::zeroMemory(pointer);
2541 2542
}

2543
PointerType PointerBuilder::getPointerType() const {
2544 2545 2546 2547
  if(pointer->isNull()) {
    return PointerType::NULL_;
  } else {
    WirePointer* ptr = pointer;
2548 2549
    SegmentBuilder* sgmt = segment;
    WireHelpers::followFars(ptr, ptr->target(), sgmt);
2550
    switch(ptr->kind()) {
2551 2552 2553 2554 2555 2556 2557 2558 2559
      case WirePointer::FAR:
        KJ_FAIL_ASSERT("far pointer not followed?");
      case WirePointer::STRUCT:
        return PointerType::STRUCT;
      case WirePointer::LIST:
        return PointerType::LIST;
      case WirePointer::OTHER:
        KJ_REQUIRE(ptr->isCapability(), "unknown pointer type");
        return PointerType::CAPABILITY;
2560
    }
2561
    KJ_UNREACHABLE;
2562
  }
2563 2564
}

2565
void PointerBuilder::transferFrom(PointerBuilder other) {
2566
  if (!pointer->isNull()) {
2567
    WireHelpers::zeroObject(segment, capTable, pointer);
2568
    WireHelpers::zeroMemory(pointer);
2569
  }
2570
  WireHelpers::transferPointer(segment, pointer, other.segment, other.pointer);
2571
  WireHelpers::zeroMemory(other.pointer);
2572 2573
}

Matthew Maurer's avatar
Matthew Maurer committed
2574
void PointerBuilder::copyFrom(PointerReader other, bool canonical) {
2575 2576 2577
  if (other.pointer == nullptr) {
    if (!pointer->isNull()) {
      WireHelpers::zeroObject(segment, capTable, pointer);
2578
      WireHelpers::zeroMemory(pointer);
2579 2580 2581
    }
  } else {
    WireHelpers::copyPointer(segment, capTable, pointer,
Matthew Maurer's avatar
Matthew Maurer committed
2582 2583 2584
                             other.segment, other.capTable, other.pointer, other.nestingLimit,
                             nullptr,
                             canonical);
2585
  }
2586 2587 2588
}

PointerReader PointerBuilder::asReader() const {
2589
  return PointerReader(segment, capTable, pointer, kj::maxValue);
2590 2591
}

2592 2593
BuilderArena* PointerBuilder::getArena() const {
  return segment->getArena();
2594 2595
}

2596 2597 2598 2599 2600 2601 2602 2603 2604 2605
CapTableBuilder* PointerBuilder::getCapTable() {
  return capTable;
}

PointerBuilder PointerBuilder::imbue(CapTableBuilder* capTable) {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

2606 2607 2608
// =======================================================================================
// PointerReader

2609 2610
PointerReader PointerReader::getRoot(SegmentReader* segment, CapTableReader* capTable,
                                     const word* location, int nestingLimit) {
Kenton Varda's avatar
Kenton Varda committed
2611 2612 2613 2614 2615
  KJ_REQUIRE(WireHelpers::boundsCheck(segment, location, location + POINTER_SIZE_IN_WORDS),
             "Root location out-of-bounds.") {
    location = nullptr;
  }

2616 2617
  return PointerReader(segment, capTable,
      reinterpret_cast<const WirePointer*>(location), nestingLimit);
Kenton Varda's avatar
Kenton Varda committed
2618 2619
}

2620 2621
StructReader PointerReader::getStruct(const word* defaultValue) const {
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
2622
  return WireHelpers::readStructPointer(segment, capTable, ref, defaultValue, nestingLimit);
2623 2624
}

2625
ListReader PointerReader::getList(ElementSize expectedElementSize, const word* defaultValue) const {
2626 2627
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
  return WireHelpers::readListPointer(
2628
      segment, capTable, ref, defaultValue, expectedElementSize, nestingLimit);
2629 2630
}

2631 2632 2633
ListReader PointerReader::getListAnySize(const word* defaultValue) const {
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
  return WireHelpers::readListPointer(
2634
      segment, capTable, ref, defaultValue, ElementSize::VOID /* dummy */, nestingLimit, false);
2635 2636
}

2637
template <>
2638 2639 2640
Text::Reader PointerReader::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) const {
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
  return WireHelpers::readTextPointer(segment, ref, defaultValue, defaultSize);
Kenton Varda's avatar
Kenton Varda committed
2641
}
2642

2643
template <>
2644 2645
Data::Reader PointerReader::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) const {
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
2646 2647
  return WireHelpers::readDataPointer(segment, ref, defaultValue,
      assertMaxBits<BLOB_SIZE_BITS>(defaultSize, ThrowOverflow()));
Kenton Varda's avatar
Kenton Varda committed
2648 2649
}

2650
#if !CAPNP_LITE
2651
kj::Own<ClientHook> PointerReader::getCapability() const {
2652
  const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
2653
  return WireHelpers::readCapabilityPointer(segment, capTable, ref, nestingLimit);
2654
}
2655
#endif  // !CAPNP_LITE
2656

2657 2658 2659
const word* PointerReader::getUnchecked() const {
  KJ_REQUIRE(segment == nullptr, "getUncheckedPointer() only allowed on unchecked messages.");
  return reinterpret_cast<const word*>(pointer);
2660 2661
}

2662
MessageSizeCounts PointerReader::targetSize() const {
2663
  return pointer == nullptr ? MessageSizeCounts { ZERO * WORDS, 0 }
2664
                            : WireHelpers::totalSize(segment, pointer, nestingLimit);
Kenton Varda's avatar
Kenton Varda committed
2665 2666
}

2667
PointerType PointerReader::getPointerType() const {
2668
  if(pointer == nullptr || pointer->isNull()) {
2669 2670 2671 2672 2673 2674 2675
    return PointerType::NULL_;
  } else {
    word* refTarget = nullptr;
    const WirePointer* ptr = pointer;
    SegmentReader* sgmt = segment;
    WireHelpers::followFars(ptr, refTarget, sgmt);
    switch(ptr->kind()) {
2676
      case WirePointer::FAR:
2677
        KJ_FAIL_ASSERT("far pointer not followed?") { return PointerType::NULL_; }
2678 2679 2680 2681 2682
      case WirePointer::STRUCT:
        return PointerType::STRUCT;
      case WirePointer::LIST:
        return PointerType::LIST;
      case WirePointer::OTHER:
2683
        KJ_REQUIRE(ptr->isCapability(), "unknown pointer type") { return PointerType::NULL_; }
2684
        return PointerType::CAPABILITY;
2685
    }
2686
    KJ_UNREACHABLE;
2687
  }
2688 2689
}

2690 2691 2692 2693
kj::Maybe<Arena&> PointerReader::getArena() const {
  return segment == nullptr ? nullptr : segment->getArena();
}

2694 2695 2696 2697
CapTableReader* PointerReader::getCapTable() {
  return capTable;
}

2698 2699 2700 2701 2702 2703
PointerReader PointerReader::imbue(CapTableReader* capTable) const {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

Matthew Maurer's avatar
Matthew Maurer committed
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
bool PointerReader::isCanonical(const word **readHead) {
  if (!this->pointer) {
    // The pointer is null, so we are canonical and do not read
    return true;
  }

  if (!this->pointer->isPositional()) {
    // The pointer is a FAR or OTHER pointer, and is non-canonical
    return false;
  }

  switch (this->getPointerType()) {
    case PointerType::NULL_:
      // The pointer is null, we are canonical and do not read
      return true;
2719
    case PointerType::STRUCT: {
Matthew Maurer's avatar
Matthew Maurer committed
2720
      bool dataTrunc, ptrTrunc;
2721
      auto structReader = this->getStruct(nullptr);
2722 2723
      if (structReader.getDataSectionSize() == ZERO * BITS &&
          structReader.getPointerSectionSize() == ZERO * POINTERS) {
2724 2725 2726 2727 2728
        return reinterpret_cast<const word*>(this->pointer) == structReader.getLocation();
      } else {
        return structReader.isCanonical(readHead, readHead, &dataTrunc, &ptrTrunc) && dataTrunc && ptrTrunc;
      }
    }
Matthew Maurer's avatar
Matthew Maurer committed
2729
    case PointerType::LIST:
2730
      return this->getListAnySize(nullptr).isCanonical(readHead, pointer);
Matthew Maurer's avatar
Matthew Maurer committed
2731 2732 2733 2734 2735 2736
    case PointerType::CAPABILITY:
      KJ_FAIL_ASSERT("Capabilities are not positional");
  }
  KJ_UNREACHABLE;
}

2737 2738 2739
// =======================================================================================
// StructBuilder

2740
void StructBuilder::clearAll() {
2741 2742
  if (dataSize == ONE * BITS) {
    setDataField<bool>(ONE * ELEMENTS, false);
2743
  } else {
2744
    WireHelpers::zeroMemory(reinterpret_cast<byte*>(data), dataSize / BITS_PER_BYTE);
2745 2746
  }

2747
  for (auto i: kj::zeroTo(pointerCount)) {
2748
    WireHelpers::zeroObject(segment, capTable, pointers + i);
2749
  }
2750
  WireHelpers::zeroMemory(pointers, pointerCount);
2751 2752
}

2753 2754
void StructBuilder::transferContentFrom(StructBuilder other) {
  // Determine the amount of data the builders have in common.
2755
  auto sharedDataSize = kj::min(dataSize, other.dataSize);
2756 2757 2758 2759

  if (dataSize > sharedDataSize) {
    // Since the target is larger than the source, make sure to zero out the extra bits that the
    // source doesn't have.
2760 2761
    if (dataSize == ONE * BITS) {
      setDataField<bool>(ZERO * ELEMENTS, false);
2762
    } else {
2763 2764 2765 2766
      byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE;
      // Note: this subtraction can't fail due to the if() above
      WireHelpers::zeroMemory(unshared,
          subtractChecked(dataSize, sharedDataSize, []() {}) / BITS_PER_BYTE);
2767 2768 2769 2770
    }
  }

  // Copy over the shared part.
2771 2772
  if (sharedDataSize == ONE * BITS) {
    setDataField<bool>(ZERO * ELEMENTS, other.getDataField<bool>(ZERO * ELEMENTS));
2773
  } else {
2774 2775 2776
    WireHelpers::copyMemory(reinterpret_cast<byte*>(data),
                            reinterpret_cast<byte*>(other.data),
                            sharedDataSize / BITS_PER_BYTE);
2777 2778 2779
  }

  // Zero out all pointers in the target.
2780
  for (auto i: kj::zeroTo(pointerCount)) {
2781
    WireHelpers::zeroObject(segment, capTable, pointers + i);
2782
  }
2783
  WireHelpers::zeroMemory(pointers, pointerCount);
2784 2785

  // Transfer the pointers.
2786 2787
  auto sharedPointerCount = kj::min(pointerCount, other.pointerCount);
  for (auto i: kj::zeroTo(sharedPointerCount)) {
2788 2789 2790 2791 2792 2793
    WireHelpers::transferPointer(segment, pointers + i, other.segment, other.pointers + i);
  }

  // Zero out the pointers that were transferred in the source because it no longer has ownership.
  // If the source had any extra pointers that the destination didn't have space for, we
  // intentionally leave them be, so that they'll be cleaned up later.
2794
  WireHelpers::zeroMemory(other.pointers, sharedPointerCount);
2795 2796
}

2797 2798
void StructBuilder::copyContentFrom(StructReader other) {
  // Determine the amount of data the builders have in common.
2799
  auto sharedDataSize = kj::min(dataSize, other.dataSize);
2800 2801 2802 2803

  if (dataSize > sharedDataSize) {
    // Since the target is larger than the source, make sure to zero out the extra bits that the
    // source doesn't have.
2804 2805
    if (dataSize == ONE * BITS) {
      setDataField<bool>(ZERO * ELEMENTS, false);
2806
    } else {
2807 2808 2809
      byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE;
      WireHelpers::zeroMemory(unshared,
          subtractChecked(dataSize, sharedDataSize, []() {}) / BITS_PER_BYTE);
2810 2811 2812 2813
    }
  }

  // Copy over the shared part.
2814 2815
  if (sharedDataSize == ONE * BITS) {
    setDataField<bool>(ZERO * ELEMENTS, other.getDataField<bool>(ZERO * ELEMENTS));
2816
  } else {
2817 2818 2819
    WireHelpers::copyMemory(reinterpret_cast<byte*>(data),
                            reinterpret_cast<const byte*>(other.data),
                            sharedDataSize / BITS_PER_BYTE);
2820 2821 2822
  }

  // Zero out all pointers in the target.
2823
  for (auto i: kj::zeroTo(pointerCount)) {
2824
    WireHelpers::zeroObject(segment, capTable, pointers + i);
2825
  }
2826
  WireHelpers::zeroMemory(pointers, pointerCount);
2827 2828

  // Copy the pointers.
2829 2830
  auto sharedPointerCount = kj::min(pointerCount, other.pointerCount);
  for (auto i: kj::zeroTo(sharedPointerCount)) {
2831 2832
    WireHelpers::copyPointer(segment, capTable, pointers + i,
        other.segment, other.capTable, other.pointers + i, other.nestingLimit);
2833 2834 2835
  }
}

2836
StructReader StructBuilder::asReader() const {
2837
  return StructReader(segment, capTable, data, pointers,
2838
      dataSize, pointerCount, kj::maxValue);
2839 2840
}

2841 2842 2843 2844
BuilderArena* StructBuilder::getArena() {
  return segment->getArena();
}

2845 2846 2847 2848 2849 2850 2851 2852 2853 2854
CapTableBuilder* StructBuilder::getCapTable() {
  return capTable;
}

StructBuilder StructBuilder::imbue(CapTableBuilder* capTable) {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

2855 2856 2857
// =======================================================================================
// StructReader

2858 2859 2860
MessageSizeCounts StructReader::totalSize() const {
  MessageSizeCounts result = {
    WireHelpers::roundBitsUpToWords(dataSize) + pointerCount * WORDS_PER_POINTER, 0 };
2861

2862
  for (auto i: kj::zeroTo(pointerCount)) {
2863 2864 2865 2866 2867 2868
    result += WireHelpers::totalSize(segment, pointers + i, nestingLimit);
  }

  if (segment != nullptr) {
    // This traversal should not count against the read limit, because it's highly likely that
    // the caller is going to traverse the object again, e.g. to copy it.
2869
    segment->unread(result.wordCount);
2870 2871 2872 2873 2874
  }

  return result;
}

2875
kj::Array<word> StructReader::canonicalize() {
2876
  auto size = totalSize().wordCount + POINTER_SIZE_IN_WORDS;
2877
  kj::Array<word> backing = kj::heapArray<word>(unbound(size / WORDS));
2878
  WireHelpers::zeroMemory(backing.asPtr());
2879 2880 2881 2882 2883
  FlatMessageBuilder builder(backing);
  _::PointerHelpers<AnyPointer>::getInternalBuilder(builder.initRoot<AnyPointer>()).setStruct(*this, true);
  KJ_ASSERT(builder.isCanonical());
  auto output = builder.getSegmentsForOutput()[0];
  kj::Array<word> trunc = kj::heapArray<word>(output.size());
2884
  WireHelpers::copyMemory(trunc.begin(), output);
2885 2886 2887
  return trunc;
}

2888 2889 2890 2891
CapTableReader* StructReader::getCapTable() {
  return capTable;
}

2892 2893 2894 2895 2896 2897
StructReader StructReader::imbue(CapTableReader* capTable) const {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

Matthew Maurer's avatar
Matthew Maurer committed
2898 2899 2900 2901 2902 2903 2904 2905 2906
bool StructReader::isCanonical(const word **readHead,
                               const word **ptrHead,
                               bool *dataTrunc,
                               bool *ptrTrunc) {
  if (this->getLocation() != *readHead) {
    // Our target area is not at the readHead, preorder fails
    return false;
  }

2907
  if (this->getDataSectionSize() % BITS_PER_WORD != ZERO * BITS) {
Matthew Maurer's avatar
Matthew Maurer committed
2908 2909 2910
    // Using legacy non-word-size structs, reject
    return false;
  }
2911
  auto dataSize = this->getDataSectionSize() / BITS_PER_WORD;
Matthew Maurer's avatar
Matthew Maurer committed
2912 2913

  // Mark whether the struct is properly truncated
2914
  KJ_IF_MAYBE(diff, trySubtract(dataSize, ONE * WORDS)) {
2915
    *dataTrunc = this->getDataField<uint64_t>(*diff / WORDS * ELEMENTS) != 0;
Matthew Maurer's avatar
Matthew Maurer committed
2916
  } else {
2917
    // Data segment empty.
Matthew Maurer's avatar
Matthew Maurer committed
2918 2919 2920
    *dataTrunc = true;
  }

2921
  KJ_IF_MAYBE(diff, trySubtract(this->pointerCount, ONE * POINTERS)) {
2922
    *ptrTrunc  = !this->getPointerField(*diff).isNull();
Matthew Maurer's avatar
Matthew Maurer committed
2923 2924 2925 2926 2927
  } else {
    *ptrTrunc = true;
  }

  // Advance the read head
2928
  *readHead += (dataSize + (this->pointerCount * WORDS_PER_POINTER));
Matthew Maurer's avatar
Matthew Maurer committed
2929 2930

  // Check each pointer field for canonicity
2931
  for (auto ptrIndex: kj::zeroTo(this->pointerCount)) {
Matthew Maurer's avatar
Matthew Maurer committed
2932 2933 2934 2935 2936 2937 2938 2939
    if (!this->getPointerField(ptrIndex).isCanonical(ptrHead)) {
      return false;
    }
  }

  return true;
}

2940 2941 2942
// =======================================================================================
// ListBuilder

2943
Text::Builder ListBuilder::asText() {
2944
  KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
2945
             "Expected Text, got list of non-bytes.") {
2946 2947 2948
    return Text::Builder();
  }

2949
  size_t size = unbound(elementCount / ELEMENTS);
2950

2951
  KJ_REQUIRE(size > 0, "Message contains text that is not NUL-terminated.") {
2952 2953 2954 2955 2956 2957
    return Text::Builder();
  }

  char* cptr = reinterpret_cast<char*>(ptr);
  --size;  // NUL terminator

2958
  KJ_REQUIRE(cptr[size] == '\0', "Message contains text that is not NUL-terminated.") {
2959 2960 2961 2962 2963 2964 2965
    return Text::Builder();
  }

  return Text::Builder(cptr, size);
}

Data::Builder ListBuilder::asData() {
2966
  KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
2967
             "Expected Text, got list of non-bytes.") {
2968 2969 2970
    return Data::Builder();
  }

2971
  return Data::Builder(reinterpret_cast<byte*>(ptr), unbound(elementCount / ELEMENTS));
2972 2973
}

2974
StructBuilder ListBuilder::getStructElement(ElementCount index) {
2975
  auto indexBit = upgradeBound<uint64_t>(index) * step;
2976
  byte* structData = ptr + indexBit / BITS_PER_BYTE;
2977
  KJ_DASSERT(indexBit % BITS_PER_BYTE == ZERO * BITS);
2978
  return StructBuilder(segment, capTable, structData,
2979
      reinterpret_cast<WirePointer*>(structData + structDataSize / BITS_PER_BYTE),
2980
      structDataSize, structPointerCount);
2981 2982
}

2983
ListReader ListBuilder::asReader() const {
2984
  return ListReader(segment, capTable, ptr, elementCount, step, structDataSize, structPointerCount,
2985
                    elementSize, kj::maxValue);
2986 2987
}

2988 2989 2990 2991
BuilderArena* ListBuilder::getArena() {
  return segment->getArena();
}

2992 2993 2994 2995 2996 2997 2998 2999 3000 3001
CapTableBuilder* ListBuilder::getCapTable() {
  return capTable;
}

ListBuilder ListBuilder::imbue(CapTableBuilder* capTable) {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

3002 3003 3004
// =======================================================================================
// ListReader

3005
Text::Reader ListReader::asText() {
3006
  KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
3007
             "Expected Text, got list of non-bytes.") {
3008 3009 3010
    return Text::Reader();
  }

3011
  size_t size = unbound(elementCount / ELEMENTS);
3012

3013
  KJ_REQUIRE(size > 0, "Message contains text that is not NUL-terminated.") {
3014 3015 3016 3017 3018 3019
    return Text::Reader();
  }

  const char* cptr = reinterpret_cast<const char*>(ptr);
  --size;  // NUL terminator

3020
  KJ_REQUIRE(cptr[size] == '\0', "Message contains text that is not NUL-terminated.") {
3021 3022 3023 3024 3025 3026 3027
    return Text::Reader();
  }

  return Text::Reader(cptr, size);
}

Data::Reader ListReader::asData() {
3028
  KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
3029
             "Expected Text, got list of non-bytes.") {
3030 3031 3032
    return Data::Reader();
  }

3033
  return Data::Reader(reinterpret_cast<const byte*>(ptr), unbound(elementCount / ELEMENTS));
3034 3035
}

3036
kj::ArrayPtr<const byte> ListReader::asRawBytes() {
3037
  KJ_REQUIRE(structPointerCount == ZERO * POINTERS,
3038
             "Expected data only, got pointers.") {
3039
    return kj::ArrayPtr<const byte>();
3040 3041
  }

3042 3043
  return arrayPtr(reinterpret_cast<const byte*>(ptr),
      WireHelpers::roundBitsUpToBytes(
3044
          upgradeBound<uint64_t>(elementCount) * (structDataSize / ELEMENTS)));
3045 3046
}

3047
StructReader ListReader::getStructElement(ElementCount index) const {
3048
  KJ_REQUIRE(nestingLimit > 0,
David Renshaw's avatar
David Renshaw committed
3049
             "Message is too deeply-nested or contains cycles.  See capnp::ReaderOptions.") {
3050
    return StructReader();
3051
  }
3052

3053
  auto indexBit = upgradeBound<uint64_t>(index) * step;
3054
  const byte* structData = ptr + indexBit / BITS_PER_BYTE;
3055 3056
  const WirePointer* structPointers =
      reinterpret_cast<const WirePointer*>(structData + structDataSize / BITS_PER_BYTE);
3057 3058

  // This check should pass if there are no bugs in the list pointer validation code.
3059
  KJ_DASSERT(structPointerCount == ZERO * POINTERS ||
Kenton Varda's avatar
Kenton Varda committed
3060
         (uintptr_t)structPointers % sizeof(void*) == 0,
3061
         "Pointer section of struct list element not aligned.");
3062

3063
  KJ_DASSERT(indexBit % BITS_PER_BYTE == ZERO * BITS);
3064
  return StructReader(
3065
      segment, capTable, structData, structPointers,
3066
      structDataSize, structPointerCount,
3067
      nestingLimit - 1);
3068
}
3069

3070 3071 3072 3073
CapTableReader* ListReader::getCapTable() {
  return capTable;
}

3074 3075 3076 3077 3078 3079
ListReader ListReader::imbue(CapTableReader* capTable) const {
  auto result = *this;
  result.capTable = capTable;
  return result;
}

3080
bool ListReader::isCanonical(const word **readHead, const WirePointer *ref) {
Matthew Maurer's avatar
Matthew Maurer committed
3081 3082 3083 3084 3085 3086 3087 3088
  switch (this->getElementSize()) {
    case ElementSize::INLINE_COMPOSITE: {
      *readHead += 1;
      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
        // The next word to read is the tag word, but the pointer is in
        // front of it, so our check is slightly different
        return false;
      }
3089
      if (this->structDataSize % BITS_PER_WORD != ZERO * BITS) {
Matthew Maurer's avatar
Matthew Maurer committed
3090 3091
        return false;
      }
3092 3093
      auto elementSize = StructSize(this->structDataSize / BITS_PER_WORD,
                                    this->structPointerCount).total() / ELEMENTS;
3094
      auto totalSize = upgradeBound<uint64_t>(this->elementCount) * elementSize;
3095
      if (totalSize != ref->listRef.inlineCompositeWordCount()) {
3096 3097
        return false;
      }
3098
      if (elementSize == ZERO * WORDS / ELEMENTS) {
3099 3100
        return true;
      }
3101
      auto listEnd = *readHead + totalSize;
Matthew Maurer's avatar
Matthew Maurer committed
3102 3103 3104
      auto pointerHead = listEnd;
      bool listDataTrunc = false;
      bool listPtrTrunc = false;
3105
      for (auto ec: kj::zeroTo(this->elementCount)) {
Matthew Maurer's avatar
Matthew Maurer committed
3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
        bool dataTrunc, ptrTrunc;
        if (!this->getStructElement(ec).isCanonical(readHead,
                                                    &pointerHead,
                                                    &dataTrunc,
                                                    &ptrTrunc)) {
          return false;
        }
        listDataTrunc |= dataTrunc;
        listPtrTrunc  |= ptrTrunc;
      }
      KJ_REQUIRE(*readHead == listEnd, *readHead, listEnd);
      *readHead = pointerHead;
      return listDataTrunc && listPtrTrunc;
    }
    case ElementSize::POINTER: {
      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
        return false;
      }
3124 3125
      *readHead += this->elementCount * (POINTERS / ELEMENTS) * WORDS_PER_POINTER;
      for (auto ec: kj::zeroTo(this->elementCount)) {
Matthew Maurer's avatar
Matthew Maurer committed
3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136
        if (!this->getPointerElement(ec).isCanonical(readHead)) {
          return false;
        }
      }
      return true;
    }
    default: {
      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
        return false;
      }

3137
      auto bitSize = upgradeBound<uint64_t>(this->elementCount) *
Matthew Maurer's avatar
Matthew Maurer committed
3138
                     dataBitsPerElement(this->elementSize);
3139
      *readHead += WireHelpers::roundBitsUpToWords(bitSize);
Matthew Maurer's avatar
Matthew Maurer committed
3140 3141 3142 3143 3144 3145
      return true;
    }
  }
  KJ_UNREACHABLE;
}

3146 3147 3148
// =======================================================================================
// OrphanBuilder

3149 3150
OrphanBuilder OrphanBuilder::initStruct(
    BuilderArena* arena, CapTableBuilder* capTable, StructSize size) {
3151
  OrphanBuilder result;
3152 3153
  StructBuilder builder = WireHelpers::initStructPointer(
      result.tagAsPtr(), nullptr, capTable, size, arena);
3154
  result.segment = builder.segment;
3155
  result.capTable = capTable;
3156
  result.location = builder.getLocation();
3157 3158 3159 3160
  return result;
}

OrphanBuilder OrphanBuilder::initList(
3161 3162
    BuilderArena* arena, CapTableBuilder* capTable,
    ElementCount elementCount, ElementSize elementSize) {
3163
  OrphanBuilder result;
3164
  ListBuilder builder = WireHelpers::initListPointer(
3165
      result.tagAsPtr(), nullptr, capTable, elementCount, elementSize, arena);
3166
  result.segment = builder.segment;
3167
  result.capTable = capTable;
3168
  result.location = builder.getLocation();
3169 3170 3171 3172
  return result;
}

OrphanBuilder OrphanBuilder::initStructList(
3173 3174
    BuilderArena* arena, CapTableBuilder* capTable,
    ElementCount elementCount, StructSize elementSize) {
3175 3176
  OrphanBuilder result;
  ListBuilder builder = WireHelpers::initStructListPointer(
3177
      result.tagAsPtr(), nullptr, capTable, elementCount, elementSize, arena);
3178
  result.segment = builder.segment;
3179
  result.capTable = capTable;
3180 3181
  result.location = builder.getLocation();
  return result;
3182 3183
}

3184 3185
OrphanBuilder OrphanBuilder::initText(
    BuilderArena* arena, CapTableBuilder* capTable, ByteCount size) {
3186
  OrphanBuilder result;
3187 3188
  auto allocation = WireHelpers::initTextPointer(result.tagAsPtr(), nullptr, capTable,
      assertMax<MAX_TEXT_SIZE>(size, ThrowOverflow()), arena);
3189
  result.segment = allocation.segment;
3190
  result.capTable = capTable;
3191
  result.location = reinterpret_cast<word*>(allocation.value.begin());
3192 3193 3194
  return result;
}

3195 3196
OrphanBuilder OrphanBuilder::initData(
    BuilderArena* arena, CapTableBuilder* capTable, ByteCount size) {
3197
  OrphanBuilder result;
3198 3199
  auto allocation = WireHelpers::initDataPointer(result.tagAsPtr(), nullptr, capTable,
      assertMaxBits<BLOB_SIZE_BITS>(size), arena);
3200
  result.segment = allocation.segment;
3201
  result.capTable = capTable;
3202
  result.location = reinterpret_cast<word*>(allocation.value.begin());
3203 3204 3205
  return result;
}

3206 3207
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, StructReader copyFrom) {
3208
  OrphanBuilder result;
3209 3210
  auto allocation = WireHelpers::setStructPointer(
      nullptr, capTable, result.tagAsPtr(), copyFrom, arena);
3211
  result.segment = allocation.segment;
3212
  result.capTable = capTable;
3213
  result.location = reinterpret_cast<word*>(allocation.value);
3214 3215 3216
  return result;
}

3217 3218
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, ListReader copyFrom) {
3219
  OrphanBuilder result;
3220 3221
  auto allocation = WireHelpers::setListPointer(
      nullptr, capTable, result.tagAsPtr(), copyFrom, arena);
3222
  result.segment = allocation.segment;
3223
  result.capTable = capTable;
3224
  result.location = reinterpret_cast<word*>(allocation.value);
3225 3226 3227
  return result;
}

3228 3229
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, PointerReader copyFrom) {
3230
  OrphanBuilder result;
3231
  auto allocation = WireHelpers::copyPointer(
3232 3233
      nullptr, capTable, result.tagAsPtr(),
      copyFrom.segment, copyFrom.capTable, copyFrom.pointer, copyFrom.nestingLimit, arena);
3234
  result.segment = allocation.segment;
3235
  result.capTable = capTable;
3236 3237 3238 3239
  result.location = reinterpret_cast<word*>(allocation.value);
  return result;
}

3240 3241
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, Text::Reader copyFrom) {
3242
  OrphanBuilder result;
3243
  auto allocation = WireHelpers::setTextPointer(
3244
      result.tagAsPtr(), nullptr, capTable, copyFrom, arena);
3245
  result.segment = allocation.segment;
3246
  result.capTable = capTable;
3247
  result.location = reinterpret_cast<word*>(allocation.value.begin());
3248 3249 3250
  return result;
}

3251 3252
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, Data::Reader copyFrom) {
3253
  OrphanBuilder result;
3254
  auto allocation = WireHelpers::setDataPointer(
3255
      result.tagAsPtr(), nullptr, capTable, copyFrom, arena);
3256
  result.segment = allocation.segment;
3257
  result.capTable = capTable;
3258
  result.location = reinterpret_cast<word*>(allocation.value.begin());
3259 3260 3261
  return result;
}

3262
#if !CAPNP_LITE
3263 3264
OrphanBuilder OrphanBuilder::copy(
    BuilderArena* arena, CapTableBuilder* capTable, kj::Own<ClientHook> copyFrom) {
3265
  OrphanBuilder result;
3266
  WireHelpers::setCapabilityPointer(nullptr, capTable, result.tagAsPtr(), kj::mv(copyFrom));
3267
  result.segment = arena->getSegment(SegmentId(0));
3268
  result.capTable = capTable;
3269
  result.location = &result.tag;  // dummy to make location non-null
3270 3271
  return result;
}
3272
#endif  // !CAPNP_LITE
3273

3274 3275 3276 3277 3278 3279 3280
OrphanBuilder OrphanBuilder::concat(
    BuilderArena* arena, CapTableBuilder* capTable,
    ElementSize elementSize, StructSize structSize,
    kj::ArrayPtr<const ListReader> lists) {
  KJ_REQUIRE(lists.size() > 0, "Can't concat empty list ");

  // Find the overall element count and size.
3281
  ListElementCount elementCount = ZERO * ELEMENTS;
3282
  for (auto& list: lists) {
3283 3284
    elementCount = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(elementCount + list.elementCount,
        []() { KJ_FAIL_REQUIRE("concatenated list exceeds list size limit"); });
3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
    if (list.elementSize != elementSize) {
      // If element sizes don't all match, upgrade to struct list.
      KJ_REQUIRE(list.elementSize != ElementSize::BIT && elementSize != ElementSize::BIT,
                 "can't upgrade bit lists to struct lists");
      elementSize = ElementSize::INLINE_COMPOSITE;
    }
    structSize.data = kj::max(structSize.data,
        WireHelpers::roundBitsUpToWords(list.structDataSize));
    structSize.pointers = kj::max(structSize.pointers, list.structPointerCount);
  }

  // Allocate the list.
  OrphanBuilder result;
  ListBuilder builder = (elementSize == ElementSize::INLINE_COMPOSITE)
      ? WireHelpers::initStructListPointer(
          result.tagAsPtr(), nullptr, capTable, elementCount, structSize, arena)
      : WireHelpers::initListPointer(
          result.tagAsPtr(), nullptr, capTable, elementCount, elementSize, arena);

  // Copy elements.
  switch (elementSize) {
    case ElementSize::INLINE_COMPOSITE: {
3307
      ListElementCount pos = ZERO * ELEMENTS;
3308
      for (auto& list: lists) {
3309
        for (auto i: kj::zeroTo(list.size())) {
3310
          builder.getStructElement(pos).copyContentFrom(list.getStructElement(i));
3311 3312
          // assumeBits() safe because we checked total size earlier.
          pos = assumeBits<LIST_ELEMENT_COUNT_BITS>(pos + ONE * ELEMENTS);
3313 3314 3315 3316 3317
        }
      }
      break;
    }
    case ElementSize::POINTER: {
3318
      ListElementCount pos = ZERO * ELEMENTS;
3319
      for (auto& list: lists) {
3320
        for (auto i: kj::zeroTo(list.size())) {
3321
          builder.getPointerElement(pos).copyFrom(list.getPointerElement(i));
3322 3323
          // assumeBits() safe because we checked total size earlier.
          pos = assumeBits<LIST_ELEMENT_COUNT_BITS>(pos + ONE * ELEMENTS);
3324 3325 3326 3327 3328 3329 3330
        }
      }
      break;
    }
    case ElementSize::BIT: {
      // It's difficult to memcpy() bits since a list could start or end mid-byte. For now we
      // do a slow, naive loop. Probably no one will ever care.
3331
      ListElementCount pos = ZERO * ELEMENTS;
3332
      for (auto& list: lists) {
3333
        for (auto i: kj::zeroTo(list.size())) {
3334
          builder.setDataElement<bool>(pos, list.getDataElement<bool>(i));
3335 3336
          // assumeBits() safe because we checked total size earlier.
          pos = assumeBits<LIST_ELEMENT_COUNT_BITS>(pos + ONE * ELEMENTS);
3337 3338 3339 3340 3341
        }
      }
      break;
    }
    default: {
3342 3343 3344
      // We know all the inputs are primitives with identical size because otherwise we would have
      // chosen INLINE_COMPOSITE. Therefore, we can safely use memcpy() here instead of copying
      // each element manually.
3345 3346 3347
      byte* target = builder.ptr;
      auto step = builder.step / BITS_PER_BYTE;
      for (auto& list: lists) {
3348
        auto count = step * upgradeBound<uint64_t>(list.size());
3349
        WireHelpers::copyMemory(target, list.ptr, assumeBits<SEGMENT_WORD_COUNT_BITS>(count));
3350
        target += count;
3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362
      }
      break;
    }
  }

  // Return orphan.
  result.segment = builder.segment;
  result.capTable = capTable;
  result.location = builder.getLocation();
  return result;
}

3363 3364 3365 3366
OrphanBuilder OrphanBuilder::referenceExternalData(BuilderArena* arena, Data::Reader data) {
  KJ_REQUIRE(reinterpret_cast<uintptr_t>(data.begin()) % sizeof(void*) == 0,
             "Cannot referenceExternalData() that is not aligned.");

3367
  auto checkedSize = assertMaxBits<BLOB_SIZE_BITS>(bounded(data.size()));
3368 3369
  auto wordCount = WireHelpers::roundBytesUpToWords(checkedSize * BYTES);
  kj::ArrayPtr<const word> words(reinterpret_cast<const word*>(data.begin()),
3370
                                 unbound(wordCount / WORDS));
3371 3372 3373

  OrphanBuilder result;
  result.tagAsPtr()->setKindForOrphan(WirePointer::LIST);
3374
  result.tagAsPtr()->listRef.set(ElementSize::BYTE, checkedSize * ELEMENTS);
3375 3376
  result.segment = arena->addExternalSegment(words);

3377 3378 3379
  // External data cannot possibly contain capabilities.
  result.capTable = nullptr;

3380 3381 3382 3383 3384 3385 3386
  // const_cast OK here because we will check whether the segment is writable when we try to get
  // a builder.
  result.location = const_cast<word*>(words.begin());

  return result;
}

3387
StructBuilder OrphanBuilder::asStruct(StructSize size) {
3388 3389
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));

3390
  StructBuilder result = WireHelpers::getWritableStructPointer(
3391
      tagAsPtr(), location, segment, capTable, size, nullptr, segment->getArena());
3392 3393

  // Watch out, the pointer could have been updated if the object had to be relocated.
3394
  location = reinterpret_cast<word*>(result.data);
3395 3396 3397 3398

  return result;
}

3399
ListBuilder OrphanBuilder::asList(ElementSize elementSize) {
3400 3401
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));

3402
  ListBuilder result = WireHelpers::getWritableListPointer(
3403
      tagAsPtr(), location, segment, capTable, elementSize, nullptr, segment->getArena());
3404 3405

  // Watch out, the pointer could have been updated if the object had to be relocated.
3406 3407 3408
  // (Actually, currently this is not true for primitive lists, but let's not turn into a bug if
  // it changes!)
  location = result.getLocation();
3409 3410 3411 3412 3413

  return result;
}

ListBuilder OrphanBuilder::asStructList(StructSize elementSize) {
3414 3415
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));

3416
  ListBuilder result = WireHelpers::getWritableStructListPointer(
3417
      tagAsPtr(), location, segment, capTable, elementSize, nullptr, segment->getArena());
3418 3419

  // Watch out, the pointer could have been updated if the object had to be relocated.
3420
  location = result.getLocation();
3421 3422 3423 3424 3425

  return result;
}

Text::Builder OrphanBuilder::asText() {
3426 3427
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));

3428
  // Never relocates.
3429
  return WireHelpers::getWritableTextPointer(
3430
      tagAsPtr(), location, segment, capTable, nullptr, ZERO * BYTES);
3431 3432 3433
}

Data::Builder OrphanBuilder::asData() {
3434 3435
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));

3436
  // Never relocates.
3437
  return WireHelpers::getWritableDataPointer(
3438
      tagAsPtr(), location, segment, capTable, nullptr, ZERO * BYTES);
3439 3440
}

3441
StructReader OrphanBuilder::asStructReader(StructSize size) const {
3442
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
3443
  return WireHelpers::readStructPointer(
3444
      segment, capTable, tagAsPtr(), location, nullptr, kj::maxValue);
3445 3446
}

3447
ListReader OrphanBuilder::asListReader(ElementSize elementSize) const {
3448
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
3449
  return WireHelpers::readListPointer(
3450
      segment, capTable, tagAsPtr(), location, nullptr, elementSize, kj::maxValue);
3451 3452
}

3453
#if !CAPNP_LITE
3454
kj::Own<ClientHook> OrphanBuilder::asCapability() const {
3455
  return WireHelpers::readCapabilityPointer(segment, capTable, tagAsPtr(), kj::maxValue);
3456
}
3457
#endif  // !CAPNP_LITE
3458

3459
Text::Reader OrphanBuilder::asTextReader() const {
3460
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
3461
  return WireHelpers::readTextPointer(segment, tagAsPtr(), location, nullptr, ZERO * BYTES);
3462 3463 3464
}

Data::Reader OrphanBuilder::asDataReader() const {
3465
  KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
3466
  return WireHelpers::readDataPointer(segment, tagAsPtr(), location, nullptr, ZERO * BYTES);
3467 3468
}

3469 3470 3471
bool OrphanBuilder::truncate(ElementCount uncheckedSize, bool isText) {
  ListElementCount size = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(uncheckedSize,
      []() { KJ_FAIL_REQUIRE("requested list size is too large"); });
3472

3473 3474 3475 3476 3477
  WirePointer* ref = tagAsPtr();
  SegmentBuilder* segment = this->segment;

  word* target = WireHelpers::followFars(ref, location, segment);

3478 3479
  if (ref->isNull()) {
    // We don't know the right element size, so we can't resize this list.
3480
    return size == ZERO * ELEMENTS;
3481 3482
  }

3483
  KJ_REQUIRE(ref->kind() == WirePointer::LIST, "Can't truncate non-list.") {
3484
    return false;
3485 3486
  }

3487 3488 3489 3490 3491
  if (isText) {
    // Add space for the NUL terminator.
    size = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(size + ONE * ELEMENTS,
        []() { KJ_FAIL_REQUIRE("requested list size is too large"); });
  }
3492

3493
  auto elementSize = ref->listRef.elementSize();
3494

3495
  if (elementSize == ElementSize::INLINE_COMPOSITE) {
3496
    auto oldWordCount = ref->listRef.inlineCompositeWordCount();
3497

3498 3499 3500 3501
    WirePointer* tag = reinterpret_cast<WirePointer*>(target);
    ++target;
    KJ_REQUIRE(tag->kind() == WirePointer::STRUCT,
               "INLINE_COMPOSITE lists of non-STRUCT type are not supported.") {
3502
      return false;
3503 3504
    }
    StructSize structSize(tag->structRef.dataSize.get(), tag->structRef.ptrCount.get());
3505 3506 3507
    auto elementStep = structSize.total() / ELEMENTS;

    auto oldSize = tag->inlineCompositeListElementCount();
3508

3509
    SegmentWordCount sizeWords = assertMaxBits<SEGMENT_WORD_COUNT_BITS>(
3510
        upgradeBound<uint64_t>(size) * elementStep,
3511 3512
        []() { KJ_FAIL_ASSERT("requested list size too large to fit in message segment"); });
    SegmentWordCount oldSizeWords = assertMaxBits<SEGMENT_WORD_COUNT_BITS>(
3513
        upgradeBound<uint64_t>(oldSize) * elementStep,
3514 3515 3516
        []() { KJ_FAIL_ASSERT("prior to truncate, list is larger than max segment size?"); });

    word* newEndWord = target + sizeWords;
3517
    word* oldEndWord = target + oldWordCount;
3518

3519 3520
    if (size <= oldSize) {
      // Zero the trailing elements.
3521 3522 3523 3524
      for (auto i: kj::range(size, oldSize)) {
        // assumeBits() safe because we checked that both sizeWords and oldSizeWords are in-range
        // above.
        WireHelpers::zeroObject(segment, capTable, tag, target +
3525
            assumeBits<SEGMENT_WORD_COUNT_BITS>(upgradeBound<uint64_t>(i) * elementStep));
3526
      }
3527
      ref->listRef.setInlineComposite(sizeWords);
3528 3529 3530
      tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
      segment->tryTruncate(oldEndWord, newEndWord);
    } else if (newEndWord <= oldEndWord) {
3531
      // Apparently the old list was over-allocated? The word count is more than needed to store
3532 3533
      // the elements. This is "valid" but shouldn't happen in practice unless someone is toying
      // with us.
3534
      word* expectedEnd = target + oldSizeWords;
3535
      KJ_ASSERT(newEndWord >= expectedEnd);
3536 3537
      WireHelpers::zeroMemory(expectedEnd,
          intervalLength(expectedEnd, newEndWord, MAX_SEGMENT_WORDS));
3538 3539 3540 3541
      tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
    } else {
      if (segment->tryExtend(oldEndWord, newEndWord)) {
        // Done in-place. Nothing else to do now; the new memory is already zero'd.
3542
        ref->listRef.setInlineComposite(sizeWords);
3543 3544 3545
        tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, size);
      } else {
        // Need to re-allocate and transfer.
3546
        OrphanBuilder replacement = initStructList(segment->getArena(), capTable, size, structSize);
3547 3548

        ListBuilder newList = replacement.asStructList(structSize);
3549 3550 3551 3552
        for (auto i: kj::zeroTo(oldSize)) {
          // assumeBits() safe because we checked that both sizeWords and oldSizeWords are in-range
          // above.
          word* element = target +
3553
              assumeBits<SEGMENT_WORD_COUNT_BITS>(upgradeBound<uint64_t>(i) * elementStep);
3554
          newList.getStructElement(i).transferContentFrom(
3555
              StructBuilder(segment, capTable, element,
3556 3557 3558
                            reinterpret_cast<WirePointer*>(element + structSize.data),
                            structSize.data * BITS_PER_WORD, structSize.pointers));
        }
3559

3560 3561 3562 3563
        *this = kj::mv(replacement);
      }
    }
  } else if (elementSize == ElementSize::POINTER) {
3564 3565 3566 3567
    // TODO(cleanup): GCC won't let me declare this constexpr, claiming POINTERS is not constexpr,
    //   but it is?
    const auto POINTERS_PER_ELEMENT = ONE * POINTERS / ELEMENTS;

3568
    auto oldSize = ref->listRef.elementCount();
3569 3570
    word* newEndWord = target + size * POINTERS_PER_ELEMENT * WORDS_PER_POINTER;
    word* oldEndWord = target + oldSize * POINTERS_PER_ELEMENT * WORDS_PER_POINTER;
3571

3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585
    if (size <= oldSize) {
      // Zero the trailing elements.
      for (WirePointer* element = reinterpret_cast<WirePointer*>(newEndWord);
           element < reinterpret_cast<WirePointer*>(oldEndWord); ++element) {
        WireHelpers::zeroPointerAndFars(segment, element);
      }
      ref->listRef.set(ElementSize::POINTER, size);
      segment->tryTruncate(oldEndWord, newEndWord);
    } else {
      if (segment->tryExtend(oldEndWord, newEndWord)) {
        // Done in-place. Nothing else to do now; the new memory is already zero'd.
        ref->listRef.set(ElementSize::POINTER, size);
      } else {
        // Need to re-allocate and transfer.
3586 3587
        OrphanBuilder replacement = initList(
            segment->getArena(), capTable, size, ElementSize::POINTER);
3588 3589
        ListBuilder newList = replacement.asList(ElementSize::POINTER);
        WirePointer* oldPointers = reinterpret_cast<WirePointer*>(target);
3590 3591 3592
        for (auto i: kj::zeroTo(oldSize)) {
          newList.getPointerElement(i).transferFrom(
              PointerBuilder(segment, capTable, oldPointers + i * POINTERS_PER_ELEMENT));
3593 3594 3595 3596 3597 3598 3599
        }
        *this = kj::mv(replacement);
      }
    }
  } else {
    auto oldSize = ref->listRef.elementCount();
    auto step = dataBitsPerElement(elementSize);
3600 3601
    const auto MAX_STEP_BYTES = ONE * WORDS / ELEMENTS * BYTES_PER_WORD;
    word* newEndWord = target + WireHelpers::roundBitsUpToWords(
3602
        upgradeBound<uint64_t>(size) * step);
3603
    word* oldEndWord = target + WireHelpers::roundBitsUpToWords(
3604
        upgradeBound<uint64_t>(oldSize) * step);
3605 3606 3607 3608 3609

    if (size <= oldSize) {
      // When truncating text, we want to set the null terminator as well, so we'll do our zeroing
      // at the byte level.
      byte* begin = reinterpret_cast<byte*>(target);
3610
      byte* newEndByte = begin + WireHelpers::roundBitsUpToBytes(
3611
          upgradeBound<uint64_t>(size) * step) - isText;
3612 3613
      byte* oldEndByte = reinterpret_cast<byte*>(oldEndWord);

3614 3615
      WireHelpers::zeroMemory(newEndByte,
          intervalLength(newEndByte, oldEndByte, MAX_LIST_ELEMENTS * MAX_STEP_BYTES));
3616 3617 3618 3619 3620 3621 3622 3623 3624
      ref->listRef.set(elementSize, size);
      segment->tryTruncate(oldEndWord, newEndWord);
    } else {
      // We're trying to extend, not truncate.
      if (segment->tryExtend(oldEndWord, newEndWord)) {
        // Done in-place. Nothing else to do now; the memory is already zero'd.
        ref->listRef.set(elementSize, size);
      } else {
        // Need to re-allocate and transfer.
3625
        OrphanBuilder replacement = initList(segment->getArena(), capTable, size, elementSize);
3626
        ListBuilder newList = replacement.asList(elementSize);
3627
        auto words = WireHelpers::roundBitsUpToWords(
3628
            dataBitsPerElement(elementSize) * upgradeBound<uint64_t>(oldSize));
3629
        WireHelpers::copyMemory(reinterpret_cast<word*>(newList.ptr), target, words);
3630 3631 3632 3633
        *this = kj::mv(replacement);
      }
    }
  }
3634 3635 3636 3637 3638 3639

  return true;
}

void OrphanBuilder::truncate(ElementCount size, ElementSize elementSize) {
  if (!truncate(size, false)) {
3640 3641 3642
    // assumeBits() safe since it's checked inside truncate()
    *this = initList(segment->getArena(), capTable,
        assumeBits<LIST_ELEMENT_COUNT_BITS>(size), elementSize);
3643 3644 3645 3646 3647
  }
}

void OrphanBuilder::truncate(ElementCount size, StructSize elementSize) {
  if (!truncate(size, false)) {
3648 3649 3650
    // assumeBits() safe since it's checked inside truncate()
    *this = initStructList(segment->getArena(), capTable,
        assumeBits<LIST_ELEMENT_COUNT_BITS>(size), elementSize);
3651 3652 3653 3654 3655
  }
}

void OrphanBuilder::truncateText(ElementCount size) {
  if (!truncate(size, true)) {
3656 3657 3658
    // assumeBits() safe since it's checked inside truncate()
    *this = initText(segment->getArena(), capTable,
        assumeBits<LIST_ELEMENT_COUNT_BITS>(size) * (ONE * BYTES / ELEMENTS));
3659
  }
3660 3661
}

3662
void OrphanBuilder::euthanize() {
3663 3664 3665
  // Carefully catch any exceptions and rethrow them as recoverable exceptions since we may be in
  // a destructor.
  auto exception = kj::runCatchingExceptions([&]() {
3666
    if (tagAsPtr()->isPositional()) {
3667
      WireHelpers::zeroObject(segment, capTable, tagAsPtr(), location);
3668
    } else {
3669
      WireHelpers::zeroObject(segment, capTable, tagAsPtr());
3670
    }
3671

3672
    WireHelpers::zeroMemory(&tag, ONE * WORDS);
3673 3674 3675 3676 3677 3678 3679
    segment = nullptr;
    location = nullptr;
  });

  KJ_IF_MAYBE(e, exception) {
    kj::getExceptionCallback().onRecoverableException(kj::mv(*e));
  }
3680 3681
}

3682
}  // namespace _ (private)
3683
}  // namespace capnp