Skip to content

C
capnproto
Commit e2cbc1b5 authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

WIP integer overflow detection via template metaprogramming. 

See: https://capnproto.org/news/2015-03-02-security-advisory-and-integer-overflow-protection.html

This commit as-is is the result of wading through two years of merge conflicts. It does not build as-is because new code added in that time hasn't been converted over.
parent a37a0cc7
Hide whitespace changes
Inline Side-by-side
Showing with 852 additions and 618 deletions
c++/src/capnp/any.c++
View file @ e2cbc1b5
......@@ -45,7 +45,7 @@ kj::Own<ClientHook> AnyPointer::Reader::getPipelinedCap(
break;
case PipelineOp::Type::GET_POINTER_FIELD:
pointer = pointer.getStruct(nullptr).getPointerField(op.pointerIndex * POINTERS);
pointer = pointer.getStruct(nullptr).getPointerField(guarded(op.pointerIndex) * POINTERS);
break;
}
}
......
c++/src/capnp/any.h
View file @ e2cbc1b5
......@@ -208,9 +208,9 @@ struct AnyPointer {
// Note: Does not accept INLINE_COMPOSITE for elementSize.
inline List<AnyStruct>::Builder initAsListOfAnyStruct(
uint dataWordCount, uint pointerCount, uint elementCount);
uint16_t dataWordCount, uint16_t pointerCount, uint elementCount);
inline AnyStruct::Builder initAsAnyStruct(uint dataWordCount, uint pointerCount);
inline AnyStruct::Builder initAsAnyStruct(uint16_t dataWordCount, uint16_t pointerCount);
template <typename T>
inline void setAs(ReaderFor<T> value);
......@@ -398,10 +398,10 @@ struct List<AnyPointer, Kind::OTHER> {
inline Reader(): reader(ElementSize::POINTER) {}
inline explicit Reader(_::ListReader reader): reader(reader) {}
inline uint size() const { return reader.size() / ELEMENTS; }
inline uint size() const { return unguard(reader.size() / ELEMENTS); }
inline AnyPointer::Reader operator[](uint index) const {
KJ_IREQUIRE(index < size());
return AnyPointer::Reader(reader.getPointerElement(index * ELEMENTS));
return AnyPointer::Reader(reader.getPointerElement(guarded(index) * ELEMENTS));
}
typedef _::IndexingIterator<const Reader, typename AnyPointer::Reader> Iterator;
......@@ -430,10 +430,10 @@ struct List<AnyPointer, Kind::OTHER> {
inline operator Reader() const { return Reader(builder.asReader()); }
inline Reader asReader() const { return Reader(builder.asReader()); }
inline uint size() const { return builder.size() / ELEMENTS; }
inline uint size() const { return unguard(builder.size() / ELEMENTS); }
inline AnyPointer::Builder operator[](uint index) {
KJ_IREQUIRE(index < size());
return AnyPointer::Builder(builder.getPointerElement(index * ELEMENTS));
return AnyPointer::Builder(builder.getPointerElement(guarded(index) * ELEMENTS));
}
typedef _::IndexingIterator<Builder, typename AnyPointer::Builder> Iterator;
......@@ -563,10 +563,10 @@ public:
inline Reader(): reader(ElementSize::INLINE_COMPOSITE) {}
inline explicit Reader(_::ListReader reader): reader(reader) {}
inline uint size() const { return reader.size() / ELEMENTS; }
inline uint size() const { return unguard(reader.size() / ELEMENTS); }
inline AnyStruct::Reader operator[](uint index) const {
KJ_IREQUIRE(index < size());
return AnyStruct::Reader(reader.getStructElement(index * ELEMENTS));
return AnyStruct::Reader(reader.getStructElement(guarded(index) * ELEMENTS));
}
typedef _::IndexingIterator<const Reader, typename AnyStruct::Reader> Iterator;
......@@ -595,10 +595,10 @@ public:
inline operator Reader() const { return Reader(builder.asReader()); }
inline Reader asReader() const { return Reader(builder.asReader()); }
inline uint size() const { return builder.size() / ELEMENTS; }
inline uint size() const { return unguard(builder.size() / ELEMENTS); }
inline AnyStruct::Builder operator[](uint index) {
KJ_IREQUIRE(index < size());
return AnyStruct::Builder(builder.getStructElement(index * ELEMENTS));
return AnyStruct::Builder(builder.getStructElement(guarded(index) * ELEMENTS));
}
typedef _::IndexingIterator<Builder, typename AnyStruct::Builder> Iterator;
......@@ -628,7 +628,7 @@ public:
#endif
inline ElementSize getElementSize() { return _reader.getElementSize(); }
inline uint size() { return _reader.size() / ELEMENTS; }
inline uint size() { return unguard(_reader.size() / ELEMENTS); }
inline kj::ArrayPtr<const byte> getRawBytes() { return _reader.asRawBytes(); }
......@@ -664,7 +664,7 @@ public:
#endif
inline ElementSize getElementSize() { return _builder.getElementSize(); }
inline uint size() { return _builder.size() / ELEMENTS; }
inline uint size() { return unguard(_builder.size() / ELEMENTS); }
Equality equals(AnyList::Reader right);
inline bool operator==(AnyList::Reader right) {
......@@ -781,18 +781,21 @@ inline BuilderFor<T> AnyPointer::Builder::initAs(uint elementCount) {
inline AnyList::Builder AnyPointer::Builder::initAsAnyList(
ElementSize elementSize, uint elementCount) {
return AnyList::Builder(builder.initList(elementSize, elementCount * ELEMENTS));
return AnyList::Builder(builder.initList(elementSize, guarded(elementCount) * ELEMENTS));
}
inline List<AnyStruct>::Builder AnyPointer::Builder::initAsListOfAnyStruct(
uint dataWordCount, uint pointerCount, uint elementCount) {
return List<AnyStruct>::Builder(builder.initStructList(elementCount * ELEMENTS,
_::StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
uint16_t dataWordCount, uint16_t pointerCount, uint elementCount) {
return List<AnyStruct>::Builder(builder.initStructList(guarded(elementCount) * ELEMENTS,
_::StructSize(guarded(dataWordCount) * WORDS,
guarded(pointerCount) * POINTERS)));
}
inline AnyStruct::Builder AnyPointer::Builder::initAsAnyStruct(uint dataWordCount, uint pointerCount) {
inline AnyStruct::Builder AnyPointer::Builder::initAsAnyStruct(
uint16_t dataWordCount, uint16_t pointerCount) {
return AnyStruct::Builder(builder.initStruct(
_::StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
_::StructSize(guarded(dataWordCount) * WORDS,
guarded(pointerCount) * POINTERS)));
}
template <typename T>
......@@ -960,15 +963,16 @@ struct PointerHelpers<AnyStruct, Kind::OTHER> {
PointerBuilder builder, const word* defaultValue = nullptr) {
// TODO(someday): Allow specifying the size somehow?
return AnyStruct::Builder(builder.getStruct(
_::StructSize(0 * WORDS, 0 * POINTERS), defaultValue));
_::StructSize(ZERO * WORDS, ZERO * POINTERS), defaultValue));
}
static inline void set(PointerBuilder builder, AnyStruct::Reader value) {
builder.setStruct(value._reader);
}
static inline AnyStruct::Builder init(
PointerBuilder builder, uint dataWordCount, uint pointerCount) {
PointerBuilder builder, uint16_t dataWordCount, uint16_t pointerCount) {
return AnyStruct::Builder(builder.initStruct(
StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
StructSize(guarded(dataWordCount) * WORDS,
guarded(pointerCount) * POINTERS)));
}
// TODO(soon): implement these
......@@ -991,12 +995,15 @@ struct PointerHelpers<AnyList, Kind::OTHER> {
}
static inline AnyList::Builder init(
PointerBuilder builder, ElementSize elementSize, uint elementCount) {
return AnyList::Builder(builder.initList(elementSize, elementCount * ELEMENTS));
return AnyList::Builder(builder.initList(
elementSize, guarded(elementCount) * ELEMENTS));
}
static inline AnyList::Builder init(
PointerBuilder builder, uint dataWordCount, uint pointerCount, uint elementCount) {
PointerBuilder builder, uint16_t dataWordCount, uint16_t pointerCount, uint elementCount) {
return AnyList::Builder(builder.initStructList(
elementCount * ELEMENTS, StructSize(dataWordCount * WORDS, pointerCount * POINTERS)));
guarded(elementCount) * ELEMENTS,
StructSize(guarded(dataWordCount) * WORDS,
guarded(pointerCount) * POINTERS)));
}
// TODO(soon): implement these
......
c++/src/capnp/arena.c++
View file @ e2cbc1b5
......@@ -42,7 +42,7 @@ void ReadLimiter::unread(WordCount64 amount) {
// the limit value was not updated correctly for one or more reads, and therefore unread() could
// overflow it even if it is only unreading bytes that were actually read.
uint64_t oldValue = limit;
uint64_t newValue = oldValue + amount / WORDS;
uint64_t newValue = oldValue + unguard(amount / WORDS);
if (newValue > oldValue) {
limit = newValue;
}
......@@ -57,10 +57,24 @@ void SegmentBuilder::throwNotWritable() {
// =======================================================================================
ReaderArena::ReaderArena(MessageReader* message)
static SegmentWordCount verifySegmentSize(size_t size) {
auto gsize = guarded(size) * WORDS;
return assertMaxBits<SEGMENT_WORD_COUNT_BITS>(gsize, [&]() {
KJ_FAIL_REQUIRE("segment is too large", size);
});
}
inline ReaderArena::ReaderArena(MessageReader* message, const word* firstSegment,
SegmentWordCount firstSegmentSize)
: message(message),
readLimiter(message->getOptions().traversalLimitInWords * WORDS),
segment0(this, SegmentId(0), message->getSegment(0), &readLimiter) {}
readLimiter(guarded(message->getOptions().traversalLimitInWords) * WORDS),
segment0(this, SegmentId(0), firstSegment, firstSegmentSize, &readLimiter) {}
inline ReaderArena::ReaderArena(MessageReader* message, kj::ArrayPtr<const word> firstSegment)
: ReaderArena(message, firstSegment.begin(), verifySegmentSize(firstSegment.size())) {}
ReaderArena::ReaderArena(MessageReader* message)
: ReaderArena(message, message->getSegment(0)) {}
ReaderArena::~ReaderArena() noexcept(false) {}
......@@ -89,6 +103,8 @@ SegmentReader* ReaderArena::tryGetSegment(SegmentId id) {
return nullptr;
}
SegmentWordCount newSegmentSize = verifySegmentSize(newSegment.size());
if (*lock == nullptr) {
// OK, the segment exists, so allocate the map.
auto s = kj::heap<SegmentMap>();
......@@ -96,7 +112,8 @@ SegmentReader* ReaderArena::tryGetSegment(SegmentId id) {
*lock = kj::mv(s);
}
auto segment = kj::heap<SegmentReader>(this, id, newSegment, &readLimiter);
auto segment = kj::heap<SegmentReader>(
this, id, newSegment.begin(), newSegmentSize, &readLimiter);
SegmentReader* result = segment;
segments->insert(std::make_pair(id.value, mv(segment)));
return result;
......@@ -116,14 +133,17 @@ BuilderArena::BuilderArena(MessageBuilder* message)
BuilderArena::BuilderArena(MessageBuilder* message,
kj::ArrayPtr<MessageBuilder::SegmentInit> segments)
: message(message),
segment0(this, SegmentId(0), segments[0].space, &this->dummyLimiter, segments[0].wordsUsed) {
segment0(this, SegmentId(0), segments[0].space.begin(),
verifySegmentSize(segments[0].space.size()),
&this->dummyLimiter, verifySegmentSize(segments[0].wordsUsed)) {
if (segments.size() > 1) {
kj::Vector<kj::Own<SegmentBuilder>> builders(segments.size() - 1);
uint i = 1;
for (auto& segment: segments.slice(1, segments.size())) {
builders.add(kj::heap<SegmentBuilder>(
this, SegmentId(i++), segment.space, &this->dummyLimiter, segment.wordsUsed));
this, SegmentId(i++), segment.space.begin(), verifySegmentSize(segment.space.size()),
&this->dummyLimiter, verifySegmentSize(segment.wordsUsed)));
}
kj::Vector<kj::ArrayPtr<const word>> forOutput;
......@@ -155,15 +175,16 @@ SegmentBuilder* BuilderArena::getSegment(SegmentId id) {
}
}
BuilderArena::AllocateResult BuilderArena::allocate(WordCount amount) {
BuilderArena::AllocateResult BuilderArena::allocate(SegmentWordCount amount) {
if (segment0.getArena() == nullptr) {
// We're allocating the first segment.
kj::ArrayPtr<word> ptr = message->allocateSegment(amount / WORDS);
kj::ArrayPtr<word> ptr = message->allocateSegment(unguard(amount / WORDS));
auto actualSize = verifySegmentSize(ptr.size());
// Re-allocate segment0 in-place. This is a bit of a hack, but we have not returned any
// pointers to this segment yet, so it should be fine.
kj::dtor(segment0);
kj::ctor(segment0, this, SegmentId(0), ptr, &this->dummyLimiter);
kj::ctor(segment0, this, SegmentId(0), ptr.begin(), actualSize, &this->dummyLimiter);
segmentWithSpace = &segment0;
return AllocateResult { &segment0, segment0.allocate(amount) };
......@@ -183,7 +204,7 @@ BuilderArena::AllocateResult BuilderArena::allocate(WordCount amount) {
}
// Need to allocate a new segment.
SegmentBuilder* result = addSegmentInternal(message->allocateSegment(amount / WORDS));
SegmentBuilder* result = addSegmentInternal(message->allocateSegment(unguard(amount / WORDS)));
// Check this new segment first the next time we need to allocate.
segmentWithSpace = result;
......@@ -204,6 +225,8 @@ SegmentBuilder* BuilderArena::addSegmentInternal(kj::ArrayPtr<T> content) {
KJ_REQUIRE(segment0.getArena() != nullptr,
"Can't allocate external segments before allocating the root segment.");
auto contentSize = verifySegmentSize(content.size());
MultiSegmentState* segmentState;
KJ_IF_MAYBE(s, moreSegments) {
segmentState = *s;
......@@ -214,7 +237,8 @@ SegmentBuilder* BuilderArena::addSegmentInternal(kj::ArrayPtr<T> content) {
}
kj::Own<SegmentBuilder> newBuilder = kj::heap<SegmentBuilder>(
this, SegmentId(segmentState->builders.size() + 1), content, &this->dummyLimiter);
this, SegmentId(segmentState->builders.size() + 1),
content.begin(), contentSize, &this->dummyLimiter);
SegmentBuilder* result = newBuilder.get();
segmentState->builders.add(kj::mv(newBuilder));
......
c++/src/capnp/arena.h
View file @ e2cbc1b5
......@@ -85,7 +85,7 @@ public:
inline void reset(WordCount64 limit);
KJ_ALWAYS_INLINE(bool canRead(WordCount amount, Arena* arena));
KJ_ALWAYS_INLINE(bool canRead(WordCount64 amount, Arena* arena));
void unread(WordCount64 amount);
// Adds back some words to the limit. Useful when the caller knows they are double-reading
......@@ -113,7 +113,7 @@ public:
class SegmentReader {
public:
inline SegmentReader(Arena* arena, SegmentId id, kj::ArrayPtr<const word> ptr,
inline SegmentReader(Arena* arena, SegmentId id, const word* ptr, SegmentWordCount size,
ReadLimiter* readLimiter);
KJ_ALWAYS_INLINE(bool containsInterval(const void* from, const void* to));
......@@ -129,8 +129,8 @@ public:
inline SegmentId getSegmentId();
inline const word* getStartPtr();
inline WordCount getOffsetTo(const word* ptr);
inline WordCount getSize();
inline SegmentWordCount getOffsetTo(const word* ptr);
inline SegmentWordCount getSize();
inline kj::ArrayPtr<const word> getArray();
......@@ -140,7 +140,7 @@ public:
private:
Arena* arena;
SegmentId id;
kj::ArrayPtr<const word> ptr;
kj::ArrayPtr<const word> ptr; // size guaranteed to fit in SEGMENT_WORD_COUNT_BITS bits
ReadLimiter* readLimiter;
KJ_DISALLOW_COPY(SegmentReader);
......@@ -150,19 +150,19 @@ private:
class SegmentBuilder: public SegmentReader {
public:
inline SegmentBuilder(BuilderArena* arena, SegmentId id, kj::ArrayPtr<word> ptr,
ReadLimiter* readLimiter, size_t wordsUsed = 0);
inline SegmentBuilder(BuilderArena* arena, SegmentId id, kj::ArrayPtr<const word> ptr,
inline SegmentBuilder(BuilderArena* arena, SegmentId id, word* ptr, SegmentWordCount size,
ReadLimiter* readLimiter, SegmentWordCount wordsUsed = ZERO * WORDS);
inline SegmentBuilder(BuilderArena* arena, SegmentId id, const word* ptr, SegmentWordCount size,
ReadLimiter* readLimiter);
inline SegmentBuilder(BuilderArena* arena, SegmentId id, decltype(nullptr),
ReadLimiter* readLimiter);
KJ_ALWAYS_INLINE(word* allocate(WordCount amount));
KJ_ALWAYS_INLINE(word* allocate(SegmentWordCount amount));
KJ_ALWAYS_INLINE(void checkWritable());
// Throw an exception if the segment is read-only (meaning it is a reference to external data).
KJ_ALWAYS_INLINE(word* getPtrUnchecked(WordCount offset));
KJ_ALWAYS_INLINE(word* getPtrUnchecked(SegmentWordCount offset));
// Get a writable pointer into the segment. Throws an exception if the segment is read-only (i.e.
// a reference to external immutable data).
......@@ -210,7 +210,7 @@ public:
class ReaderArena final: public Arena {
public:
ReaderArena(MessageReader* message);
explicit ReaderArena(MessageReader* message);
~ReaderArena() noexcept(false);
KJ_DISALLOW_COPY(ReaderArena);
......@@ -234,6 +234,9 @@ private:
// TODO(perf): Thread-local thing instead? Some kind of lockless map? Or do sharing of data
// in a different way, where you have to construct a new MessageReader in each thread (but
// possibly backed by the same data)?
ReaderArena(MessageReader* message, kj::ArrayPtr<const word> firstSegment);
ReaderArena(MessageReader* message, const word* firstSegment, SegmentWordCount firstSegmentSize);
};
class BuilderArena final: public Arena {
......@@ -277,7 +280,7 @@ public:
word* words;
};
AllocateResult allocate(WordCount amount);
AllocateResult allocate(SegmentWordCount amount);
// Find a segment with at least the given amount of space available and allocate the space.
// Note that allocating directly from a particular segment is much faster, but allocating from
// the arena is guaranteed to succeed. Therefore callers should try to allocate from a specific
......@@ -339,34 +342,36 @@ private:
inline ReadLimiter::ReadLimiter()
: limit(kj::maxValue) {}
inline ReadLimiter::ReadLimiter(WordCount64 limit): limit(limit / WORDS) {}
inline ReadLimiter::ReadLimiter(WordCount64 limit): limit(unguard(limit / WORDS)) {}
inline void ReadLimiter::reset(WordCount64 limit) { this->limit = limit / WORDS; }
inline void ReadLimiter::reset(WordCount64 limit) { this->limit = unguard(limit / WORDS); }
inline bool ReadLimiter::canRead(WordCount amount, Arena* arena) {
inline bool ReadLimiter::canRead(WordCount64 amount, Arena* arena) {
// Be careful not to store an underflowed value into `limit`, even if multiple threads are
// decrementing it.
uint64_t current = limit;
if (KJ_UNLIKELY(amount / WORDS > current)) {
if (KJ_UNLIKELY(unguard(amount / WORDS) > current)) {
arena->reportReadLimitReached();
return false;
} else {
limit = current - amount / WORDS;
limit = current - unguard(amount / WORDS);
return true;
}
}
// -------------------------------------------------------------------
inline SegmentReader::SegmentReader(Arena* arena, SegmentId id, kj::ArrayPtr<const word> ptr,
ReadLimiter* readLimiter)
: arena(arena), id(id), ptr(ptr), readLimiter(readLimiter) {}
inline SegmentReader::SegmentReader(Arena* arena, SegmentId id, const word* ptr,
SegmentWordCount size, ReadLimiter* readLimiter)
: arena(arena), id(id), ptr(kj::arrayPtr(ptr, unguard(size / WORDS))),
readLimiter(readLimiter) {}
inline bool SegmentReader::containsInterval(const void* from, const void* to) {
return from >= this->ptr.begin() && to <= this->ptr.end() && from <= to &&
readLimiter->canRead(
intervalLength(reinterpret_cast<const byte*>(from),
reinterpret_cast<const byte*>(to)) / BYTES_PER_WORD,
reinterpret_cast<const byte*>(to))
/ BYTES_PER_WORD,
arena);
}
......@@ -377,31 +382,36 @@ inline bool SegmentReader::amplifiedRead(WordCount virtualAmount) {
inline Arena* SegmentReader::getArena() { return arena; }
inline SegmentId SegmentReader::getSegmentId() { return id; }
inline const word* SegmentReader::getStartPtr() { return ptr.begin(); }
inline WordCount SegmentReader::getOffsetTo(const word* ptr) {
return intervalLength(this->ptr.begin(), ptr);
inline SegmentWordCount SegmentReader::getOffsetTo(const word* ptr) {
KJ_IREQUIRE(this->ptr.begin() <= ptr && ptr < this->ptr.end());
return assumeBits<SEGMENT_WORD_COUNT_BITS>(intervalLength(this->ptr.begin(), ptr));
}
inline SegmentWordCount SegmentReader::getSize() {
return assumeBits<SEGMENT_WORD_COUNT_BITS>(ptr.size()) * WORDS;
}
inline WordCount SegmentReader::getSize() { return ptr.size() * WORDS; }
inline kj::ArrayPtr<const word> SegmentReader::getArray() { return ptr; }
inline void SegmentReader::unread(WordCount64 amount) { readLimiter->unread(amount); }
// -------------------------------------------------------------------
inline SegmentBuilder::SegmentBuilder(
BuilderArena* arena, SegmentId id, kj::ArrayPtr<word> ptr, ReadLimiter* readLimiter,
size_t wordsUsed)
: SegmentReader(arena, id, ptr, readLimiter), pos(ptr.begin() + wordsUsed), readOnly(false) {}
BuilderArena* arena, SegmentId id, word* ptr, SegmentWordCount size,
ReadLimiter* readLimiter, SegmentWordCount wordsUsed)
: SegmentReader(arena, id, ptr, size, readLimiter),
pos(ptr + wordsUsed), readOnly(false) {}
inline SegmentBuilder::SegmentBuilder(
BuilderArena* arena, SegmentId id, kj::ArrayPtr<const word> ptr, ReadLimiter* readLimiter)
: SegmentReader(arena, id, ptr, readLimiter),
BuilderArena* arena, SegmentId id, const word* ptr, SegmentWordCount size,
ReadLimiter* readLimiter)
: SegmentReader(arena, id, ptr, size, readLimiter),
// const_cast is safe here because the member won't ever be dereferenced because it appears
// to point to the end of the segment anyway.
pos(const_cast<word*>(ptr.end())),
readOnly(true) {}
pos(const_cast<word*>(ptr + size)), readOnly(true) {}
inline SegmentBuilder::SegmentBuilder(BuilderArena* arena, SegmentId id, decltype(nullptr),
ReadLimiter* readLimiter)
: SegmentReader(arena, id, nullptr, readLimiter), pos(nullptr), readOnly(false) {}
: SegmentReader(arena, id, nullptr, ZERO * WORDS, readLimiter),
pos(nullptr), readOnly(false) {}
inline word* SegmentBuilder::allocate(WordCount amount) {
inline word* SegmentBuilder::allocate(SegmentWordCount amount) {
if (intervalLength(pos, ptr.end()) < amount) {
// Not enough space in the segment for this allocation.
return nullptr;
......@@ -417,7 +427,7 @@ inline void SegmentBuilder::checkWritable() {
if (KJ_UNLIKELY(readOnly)) throwNotWritable();
}
inline word* SegmentBuilder::getPtrUnchecked(WordCount offset) {
inline word* SegmentBuilder::getPtrUnchecked(SegmentWordCount offset) {
return const_cast<word*>(ptr.begin() + offset);
}
......@@ -432,7 +442,7 @@ inline kj::ArrayPtr<const word> SegmentBuilder::currentlyAllocated() {
}
inline void SegmentBuilder::reset() {
word* start = getPtrUnchecked(0 * WORDS);
word* start = getPtrUnchecked(ZERO * WORDS);
memset(start, 0, (pos - start) * sizeof(word));
pos = start;
}
......
c++/src/capnp/capability.h
View file @ e2cbc1b5
......@@ -653,7 +653,8 @@ struct List<T, Kind::INTERFACE> {
inline uint size() const { return reader.size() / ELEMENTS; }
inline typename T::Client operator[](uint index) const {
KJ_IREQUIRE(index < size());
return typename T::Client(reader.getPointerElement(index * ELEMENTS).getCapability());
return typename T::Client(reader.getPointerElement(
guarded(index) * ELEMENTS).getCapability());
}
typedef _::IndexingIterator<const Reader, typename T::Client> Iterator;
......@@ -685,19 +686,20 @@ struct List<T, Kind::INTERFACE> {
inline uint size() const { return builder.size() / ELEMENTS; }
inline typename T::Client operator[](uint index) {
KJ_IREQUIRE(index < size());
return typename T::Client(builder.getPointerElement(index * ELEMENTS).getCapability());
return typename T::Client(builder.getPointerElement(
guarded(index) * ELEMENTS).getCapability());
}
inline void set(uint index, typename T::Client value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).setCapability(kj::mv(value.hook));
builder.getPointerElement(guarded(index) * ELEMENTS).setCapability(kj::mv(value.hook));
}
inline void adopt(uint index, Orphan<T>&& value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value));
builder.getPointerElement(guarded(index) * ELEMENTS).adopt(kj::mv(value));
}
inline Orphan<T> disown(uint index) {
KJ_IREQUIRE(index < size());
return Orphan<T>(builder.getPointerElement(index * ELEMENTS).disown());
return Orphan<T>(builder.getPointerElement(guarded(index) * ELEMENTS).disown());
}
typedef _::IndexingIterator<Builder, typename T::Client> Iterator;
......@@ -713,7 +715,7 @@ struct List<T, Kind::INTERFACE> {
private:
inline static _::ListBuilder initPointer(_::PointerBuilder builder, uint size) {
return builder.initList(ElementSize::POINTER, size * ELEMENTS);
return builder.initList(ElementSize::POINTER, guarded(size) * ELEMENTS);
}
inline static _::ListBuilder getFromPointer(_::PointerBuilder builder, const word* defaultValue) {
return builder.getList(ElementSize::POINTER, defaultValue);
......
c++/src/capnp/common.h
View file @ e2cbc1b5
......@@ -345,138 +345,176 @@ static_assert(sizeof(word) == 8, "uint64_t is not 8 bytes?");
namespace _ { class BitLabel; class ElementLabel; struct WirePointer; }
typedef kj::Quantity<uint, _::BitLabel> BitCount;
typedef kj::Quantity<uint8_t, _::BitLabel> BitCount8;
typedef kj::Quantity<uint16_t, _::BitLabel> BitCount16;
typedef kj::Quantity<uint32_t, _::BitLabel> BitCount32;
typedef kj::Quantity<uint64_t, _::BitLabel> BitCount64;
typedef kj::Quantity<uint, byte> ByteCount;
typedef kj::Quantity<uint8_t, byte> ByteCount8;
typedef kj::Quantity<uint16_t, byte> ByteCount16;
typedef kj::Quantity<uint32_t, byte> ByteCount32;
typedef kj::Quantity<uint64_t, byte> ByteCount64;
typedef kj::Quantity<uint, word> WordCount;
typedef kj::Quantity<uint8_t, word> WordCount8;
typedef kj::Quantity<uint16_t, word> WordCount16;
typedef kj::Quantity<uint32_t, word> WordCount32;
typedef kj::Quantity<uint64_t, word> WordCount64;
typedef kj::Quantity<uint, _::ElementLabel> ElementCount;
typedef kj::Quantity<uint8_t, _::ElementLabel> ElementCount8;
typedef kj::Quantity<uint16_t, _::ElementLabel> ElementCount16;
typedef kj::Quantity<uint32_t, _::ElementLabel> ElementCount32;
typedef kj::Quantity<uint64_t, _::ElementLabel> ElementCount64;
typedef kj::Quantity<uint, _::WirePointer> WirePointerCount;
typedef kj::Quantity<uint8_t, _::WirePointer> WirePointerCount8;
typedef kj::Quantity<uint16_t, _::WirePointer> WirePointerCount16;
typedef kj::Quantity<uint32_t, _::WirePointer> WirePointerCount32;
typedef kj::Quantity<uint64_t, _::WirePointer> WirePointerCount64;
template <uint width, typename T = uint>
using BitCountN = kj::Quantity<kj::Guarded<kj::maxValueForBits<width>(), T>, _::BitLabel>;
template <uint width, typename T = uint>
using ByteCountN = kj::Quantity<kj::Guarded<kj::maxValueForBits<width>(), T>, byte>;
template <uint width, typename T = uint>
using WordCountN = kj::Quantity<kj::Guarded<kj::maxValueForBits<width>(), T>, word>;
template <uint width, typename T = uint>
using ElementCountN = kj::Quantity<kj::Guarded<kj::maxValueForBits<width>(), T>, _::ElementLabel>;
template <uint width, typename T = uint>
using WirePointerCountN = kj::Quantity<kj::Guarded<kj::maxValueForBits<width>(), T>, _::WirePointer>;
typedef BitCountN<8, uint8_t> BitCount8;
typedef BitCountN<16, uint16_t> BitCount16;
typedef BitCountN<32, uint32_t> BitCount32;
typedef BitCountN<64, uint64_t> BitCount64;
typedef BitCountN<sizeof(uint) * 8, uint> BitCount;
typedef ByteCountN<8, uint8_t> ByteCount8;
typedef ByteCountN<16, uint16_t> ByteCount16;
typedef ByteCountN<32, uint32_t> ByteCount32;
typedef ByteCountN<64, uint64_t> ByteCount64;
typedef ByteCountN<sizeof(uint) * 8, uint> ByteCount;
typedef WordCountN<8, uint8_t> WordCount8;
typedef WordCountN<16, uint16_t> WordCount16;
typedef WordCountN<32, uint32_t> WordCount32;
typedef WordCountN<64, uint64_t> WordCount64;
typedef WordCountN<sizeof(uint) * 8, uint> WordCount;
typedef ElementCountN<8, uint8_t> ElementCount8;
typedef ElementCountN<16, uint16_t> ElementCount16;
typedef ElementCountN<32, uint32_t> ElementCount32;
typedef ElementCountN<64, uint64_t> ElementCount64;
typedef ElementCountN<sizeof(uint) * 8, uint> ElementCount;
typedef WirePointerCountN<8, uint8_t> WirePointerCount8;
typedef WirePointerCountN<16, uint16_t> WirePointerCount16;
typedef WirePointerCountN<32, uint32_t> WirePointerCount32;
typedef WirePointerCountN<64, uint64_t> WirePointerCount64;
typedef WirePointerCountN<sizeof(uint) * 8, uint> WirePointerCount;
template <uint width>
using BitsPerElementN = decltype(BitCountN<width>() / ElementCountN<width>());
template <uint width>
using BytesPerElementN = decltype(ByteCountN<width>() / ElementCountN<width>());
template <uint width>
using WordsPerElementN = decltype(WordCountN<width>() / ElementCountN<width>());
template <uint width>
using PointersPerElementN = decltype(WirePointerCountN<width>() / ElementCountN<width>());
using kj::guarded;
using kj::unguard;
using kj::unguardAs;
using kj::unguardMax;
using kj::unguardMaxBits;
using kj::assertMax;
using kj::assertMaxBits;
using kj::upgradeGuard;
using kj::ThrowOverflow;
using kj::assumeBits;
using kj::subtractChecked;
template <typename T, typename U>
inline constexpr U* operator+(U* ptr, kj::Quantity<T, U> offset) {
return ptr + offset / kj::unit<kj::Quantity<T, U>>();
return ptr + unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr const U* operator+(const U* ptr, kj::Quantity<T, U> offset) {
return ptr + offset / kj::unit<kj::Quantity<T, U>>();
return ptr + unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr U* operator+=(U*& ptr, kj::Quantity<T, U> offset) {
return ptr = ptr + offset / kj::unit<kj::Quantity<T, U>>();
return ptr = ptr + unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr const U* operator+=(const U*& ptr, kj::Quantity<T, U> offset) {
return ptr = ptr + offset / kj::unit<kj::Quantity<T, U>>();
return ptr = ptr + unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr U* operator-(U* ptr, kj::Quantity<T, U> offset) {
return ptr - offset / kj::unit<kj::Quantity<T, U>>();
return ptr - unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr const U* operator-(const U* ptr, kj::Quantity<T, U> offset) {
return ptr - offset / kj::unit<kj::Quantity<T, U>>();
return ptr - unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr U* operator-=(U*& ptr, kj::Quantity<T, U> offset) {
return ptr = ptr - offset / kj::unit<kj::Quantity<T, U>>();
return ptr = ptr - unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
template <typename T, typename U>
inline constexpr const U* operator-=(const U*& ptr, kj::Quantity<T, U> offset) {
return ptr = ptr - offset / kj::unit<kj::Quantity<T, U>>();
return ptr = ptr - unguard(offset / kj::unit<kj::Quantity<T, U>>());
}
#else
typedef uint BitCount;
typedef uint8_t BitCount8;
typedef uint16_t BitCount16;
typedef uint32_t BitCount32;
typedef uint64_t BitCount64;
typedef uint ByteCount;
typedef uint8_t ByteCount8;
typedef uint16_t ByteCount16;
typedef uint32_t ByteCount32;
typedef uint64_t ByteCount64;
typedef uint WordCount;
typedef uint8_t WordCount8;
typedef uint16_t WordCount16;
typedef uint32_t WordCount32;
typedef uint64_t WordCount64;
typedef uint ElementCount;
typedef uint8_t ElementCount8;
typedef uint16_t ElementCount16;
typedef uint32_t ElementCount32;
typedef uint64_t ElementCount64;
typedef uint WirePointerCount;
typedef uint8_t WirePointerCount8;
typedef uint16_t WirePointerCount16;
typedef uint32_t WirePointerCount32;
typedef uint64_t WirePointerCount64;
#endif
constexpr auto BITS = kj::unit<BitCountN<1>>();
constexpr auto BYTES = kj::unit<ByteCountN<1>>();
constexpr auto WORDS = kj::unit<WordCountN<1>>();
constexpr auto ELEMENTS = kj::unit<ElementCountN<1>>();
constexpr auto POINTERS = kj::unit<WirePointerCountN<1>>();
constexpr BitCount BITS = kj::unit<BitCount>();
constexpr ByteCount BYTES = kj::unit<ByteCount>();
constexpr WordCount WORDS = kj::unit<WordCount>();
constexpr ElementCount ELEMENTS = kj::unit<ElementCount>();
constexpr WirePointerCount POINTERS = kj::unit<WirePointerCount>();
constexpr auto ZERO = kj::guarded<0>();
constexpr auto ONE = kj::guarded<1>();
// GCC 4.7 actually gives unused warnings on these constants in opt mode...
constexpr auto BITS_PER_BYTE KJ_UNUSED = 8 * BITS / BYTES;
constexpr auto BITS_PER_WORD KJ_UNUSED = 64 * BITS / WORDS;
constexpr auto BYTES_PER_WORD KJ_UNUSED = 8 * BYTES / WORDS;
constexpr auto BITS_PER_POINTER KJ_UNUSED = 64 * BITS / POINTERS;
constexpr auto BYTES_PER_POINTER KJ_UNUSED = 8 * BYTES / POINTERS;
constexpr auto WORDS_PER_POINTER KJ_UNUSED = 1 * WORDS / POINTERS;
constexpr WordCount POINTER_SIZE_IN_WORDS = 1 * POINTERS * WORDS_PER_POINTER;
constexpr auto BITS_PER_BYTE KJ_UNUSED = guarded<8>() * BITS / BYTES;
constexpr auto BITS_PER_WORD KJ_UNUSED = guarded<64>() * BITS / WORDS;
constexpr auto BYTES_PER_WORD KJ_UNUSED = guarded<8>() * BYTES / WORDS;
constexpr auto BITS_PER_POINTER KJ_UNUSED = guarded<64>() * BITS / POINTERS;
constexpr auto BYTES_PER_POINTER KJ_UNUSED = guarded<8>() * BYTES / POINTERS;
constexpr auto WORDS_PER_POINTER KJ_UNUSED = ONE * WORDS / POINTERS;
constexpr auto POINTER_SIZE_IN_WORDS = ONE * POINTERS * WORDS_PER_POINTER;
constexpr uint SEGMENT_WORD_COUNT_BITS = 29; // Number of words in a segment.
constexpr uint LIST_ELEMENT_COUNT_BITS = 29; // Number of elements in a list.
constexpr uint STRUCT_DATA_WORD_COUNT_BITS = 16; // Number of words in a Struct data section.
constexpr uint STRUCT_POINTER_COUNT_BITS = 16; // Number of pointers in a Struct pointer section.
constexpr uint BLOB_SIZE_BITS = 29; // Number of bytes in a blob.
typedef WordCountN<SEGMENT_WORD_COUNT_BITS> SegmentWordCount;
typedef ElementCountN<LIST_ELEMENT_COUNT_BITS> ListElementCount;
typedef WordCountN<STRUCT_DATA_WORD_COUNT_BITS, uint16_t> StructDataWordCount;
typedef WirePointerCountN<STRUCT_POINTER_COUNT_BITS, uint16_t> StructPointerCount;
typedef ByteCountN<BLOB_SIZE_BITS> BlobSize;
constexpr auto MAX_SEGMENT_WORDS =
guarded<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>()>() * WORDS;
constexpr auto MAX_LIST_ELEMENTS =
guarded<kj::maxValueForBits<LIST_ELEMENT_COUNT_BITS>()>() * ELEMENTS;
constexpr auto MAX_STUCT_DATA_WORDS =
guarded<kj::maxValueForBits<STRUCT_DATA_WORD_COUNT_BITS>()>() * WORDS;
constexpr auto MAX_STRUCT_POINTER_COUNT =
guarded<kj::maxValueForBits<STRUCT_POINTER_COUNT_BITS>()>() *POINTERS;
using StructDataBitCount = decltype(WordCountN<STRUCT_POINTER_COUNT_BITS>() * BITS_PER_WORD);
using StructDataElementOffset = decltype(StructDataBitCount() * (ONE * ELEMENTS / BITS));
// Number of bits in a Struct data segment (should come out to BitCountN<22>).
constexpr uint MAX_TEXT_SIZE = kj::maxValueForBits<BLOB_SIZE_BITS>() - 1;
typedef kj::Quantity<kj::Guarded<MAX_TEXT_SIZE, uint>, byte> TextSize;
// Not including NUL terminator.
template <typename T>
inline KJ_CONSTEXPR() decltype(BYTES / ELEMENTS) bytesPerElement() {
return sizeof(T) * BYTES / ELEMENTS;
return guarded<sizeof(T)>() * BYTES / ELEMENTS;
}
template <typename T>
inline KJ_CONSTEXPR() decltype(BITS / ELEMENTS) bitsPerElement() {
return sizeof(T) * 8 * BITS / ELEMENTS;
return guarded<sizeof(T)>() * 8 * BITS / ELEMENTS;
}
inline constexpr ByteCount intervalLength(const byte* a, const byte* b) {
return uint(b - a) * BYTES;
inline constexpr ByteCountN<sizeof(size_t) * 8, size_t>
intervalLength(const byte* a, const byte* b) {
return kj::guarded(b - a) * BYTES;
}
inline constexpr WordCount intervalLength(const word* a, const word* b) {
return uint(b - a) * WORDS;
inline constexpr WordCountN<sizeof(size_t) * 8, size_t>
intervalLength(const word* a, const word* b) {
return kj::guarded(b - a) * WORDS;
}
#else
#error TODO
#endif
} // namespace capnp
#endif // CAPNP_COMMON_H_
c++/src/capnp/layout.c++
View file @ e2cbc1b5
......@@ -59,6 +59,8 @@ namespace _ { // private
#endif // !CAPNP_LITE
#define G(n) guarded<n>()
// =======================================================================================
struct WirePointer {
......@@ -159,26 +161,28 @@ struct WirePointer {
offsetAndKind.set(kind | 0xfffffffc);
}
KJ_ALWAYS_INLINE(ElementCount inlineCompositeListElementCount() const) {
return (offsetAndKind.get() >> 2) * ELEMENTS;
KJ_ALWAYS_INLINE(ListElementCount inlineCompositeListElementCount() const) {
return ((guarded(offsetAndKind.get()) >> G(2))
& G(kj::maxValueForBits<LIST_ELEMENT_COUNT_BITS>())) * ELEMENTS;
}
KJ_ALWAYS_INLINE(void setKindAndInlineCompositeListElementCount(
Kind kind, ElementCount elementCount)) {
offsetAndKind.set(((elementCount / ELEMENTS) << 2) | kind);
Kind kind, ListElementCount elementCount)) {
offsetAndKind.set(unguardAs<uint32_t>((elementCount / ELEMENTS) << G(2)) | kind);
}
KJ_ALWAYS_INLINE(WordCount farPositionInSegment() const) {
KJ_ALWAYS_INLINE(SegmentWordCount farPositionInSegment() const) {
KJ_DREQUIRE(kind() == FAR,
"positionInSegment() should only be called on FAR pointers.");
return (offsetAndKind.get() >> 3) * WORDS;
return (guarded(offsetAndKind.get()) >> G(3)) * WORDS;
}
KJ_ALWAYS_INLINE(bool isDoubleFar() const) {
KJ_DREQUIRE(kind() == FAR,
"isDoubleFar() should only be called on FAR pointers.");
return (offsetAndKind.get() >> 2) & 1;
return unguard((guarded(offsetAndKind.get()) >> G(2)) & G(1));
}
KJ_ALWAYS_INLINE(void setFar(bool isDoubleFar, WordCount pos)) {
offsetAndKind.set(((pos / WORDS) << 3) | (static_cast<uint32_t>(isDoubleFar) << 2) |
KJ_ALWAYS_INLINE(void setFar(bool isDoubleFar, WordCountN<29> pos)) {
offsetAndKind.set(unguardAs<uint32_t>((pos / WORDS) << G(3)) |
(static_cast<uint32_t>(isDoubleFar) << 2) |
static_cast<uint32_t>(Kind::FAR));
}
KJ_ALWAYS_INLINE(void setCap(uint index)) {
......@@ -196,11 +200,11 @@ struct WirePointer {
WireValue<WordCount16> dataSize;
WireValue<WirePointerCount16> ptrCount;
inline WordCount wordSize() const {
return dataSize.get() + ptrCount.get() * WORDS_PER_POINTER;
inline WordCountN<17> wordSize() const {
return upgradeGuard<uint32_t>(dataSize.get()) + ptrCount.get() * WORDS_PER_POINTER;
}
KJ_ALWAYS_INLINE(void set(WordCount ds, WirePointerCount rc)) {
KJ_ALWAYS_INLINE(void set(WordCount16 ds, WirePointerCount16 rc)) {
dataSize.set(ds);
ptrCount.set(rc);
}
......@@ -216,21 +220,20 @@ struct WirePointer {
KJ_ALWAYS_INLINE(ElementSize elementSize() const) {
return static_cast<ElementSize>(elementSizeAndCount.get() & 7);
}
KJ_ALWAYS_INLINE(ElementCount elementCount() const) {
return (elementSizeAndCount.get() >> 3) * ELEMENTS;
KJ_ALWAYS_INLINE(ElementCountN<29> elementCount() const) {
return (guarded(elementSizeAndCount.get()) >> G(3)) * ELEMENTS;
}
KJ_ALWAYS_INLINE(WordCount inlineCompositeWordCount() const) {
return elementCount() * (1 * WORDS / ELEMENTS);
KJ_ALWAYS_INLINE(WordCountN<29> inlineCompositeWordCount() const) {
return elementCount() * (ONE * WORDS / ELEMENTS);
}
KJ_ALWAYS_INLINE(void set(ElementSize es, ElementCount ec)) {
KJ_DREQUIRE(ec < (1 << 29) * ELEMENTS, "Lists are limited to 2**29 elements.");
elementSizeAndCount.set(((ec / ELEMENTS) << 3) | static_cast<int>(es));
KJ_ALWAYS_INLINE(void set(ElementSize es, ElementCountN<29> ec)) {
elementSizeAndCount.set(unguardAs<uint32_t>((ec / ELEMENTS) << G(3)) |
static_cast<int>(es));
}
KJ_ALWAYS_INLINE(void setInlineComposite(WordCount wc)) {
KJ_DREQUIRE(wc < (1 << 29) * WORDS, "Inline composite lists are limited to 2**29 words.");
elementSizeAndCount.set(((wc / WORDS) << 3) |
KJ_ALWAYS_INLINE(void setInlineComposite(WordCountN<29> wc)) {
elementSizeAndCount.set(unguardAs<uint32_t>((wc / WORDS) << G(3)) |
static_cast<int>(ElementSize::INLINE_COMPOSITE));
}
};
......@@ -269,17 +272,19 @@ struct WirePointer {
};
static_assert(sizeof(WirePointer) == sizeof(word),
"capnp::WirePointer is not exactly one word. This will probably break everything.");
static_assert(POINTERS * WORDS_PER_POINTER * BYTES_PER_WORD / BYTES == sizeof(WirePointer),
static_assert(unguardAs<size_t>(POINTERS * WORDS_PER_POINTER * BYTES_PER_WORD / BYTES) ==
sizeof(WirePointer),
"WORDS_PER_POINTER is wrong.");
static_assert(POINTERS * BYTES_PER_POINTER / BYTES == sizeof(WirePointer),
static_assert(unguardAs<size_t>(POINTERS * BYTES_PER_POINTER / BYTES) == sizeof(WirePointer),
"BYTES_PER_POINTER is wrong.");
static_assert(POINTERS * BITS_PER_POINTER / BITS_PER_BYTE / BYTES == sizeof(WirePointer),
static_assert(unguardAs<size_t>(POINTERS * BITS_PER_POINTER / BITS_PER_BYTE / BYTES) ==
sizeof(WirePointer),
"BITS_PER_POINTER is wrong.");
namespace {
static const union {
AlignedData<POINTER_SIZE_IN_WORDS / WORDS> word;
AlignedData<unguard(POINTER_SIZE_IN_WORDS / WORDS)> word;
WirePointer pointer;
} zero = {{{0}}};
......@@ -298,22 +303,72 @@ struct SegmentAnd {
} // namespace
struct WireHelpers {
#if CAPNP_DEBUG_TYPES
template <uint64_t maxN, typename T>
static KJ_ALWAYS_INLINE(
kj::Quantity<kj::Guarded<(maxN + 7) / 8, T>, word> roundBytesUpToWords(
kj::Quantity<kj::Guarded<maxN, T>, byte> bytes)) {
static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
return (bytes + G(7) * BYTES) / BYTES_PER_WORD;
}
template <uint64_t maxN, typename T>
static KJ_ALWAYS_INLINE(
kj::Quantity<kj::Guarded<(maxN + 7) / 8, T>, byte> roundBitsUpToBytes(
kj::Quantity<kj::Guarded<maxN, T>, BitLabel> bits)) {
return (bits + G(7) * BITS) / BITS_PER_BYTE;
}
template <uint64_t maxN, typename T>
static KJ_ALWAYS_INLINE(
kj::Quantity<kj::Guarded<(maxN + 63) / 64, T>, word> roundBitsUpToWords(
kj::Quantity<kj::Guarded<maxN, T>, BitLabel> bits)) {
static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
return (bits + G(63) * BITS) / BITS_PER_WORD;
}
#else
static KJ_ALWAYS_INLINE(WordCount roundBytesUpToWords(ByteCount bytes)) {
static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
return (bytes + 7 * BYTES) / BYTES_PER_WORD;
return (bytes + G(7) * BYTES) / BYTES_PER_WORD;
}
static KJ_ALWAYS_INLINE(ByteCount roundBitsUpToBytes(BitCount bits)) {
return (bits + 7 * BITS) / BITS_PER_BYTE;
return (bits + G(7) * BITS) / BITS_PER_BYTE;
}
static KJ_ALWAYS_INLINE(WordCount64 roundBitsUpToWords(BitCount64 bits)) {
static_assert(sizeof(word) == 8, "This code assumes 64-bit words.");
return (bits + 63 * BITS) / BITS_PER_WORD;
return (bits + G(63) * BITS) / BITS_PER_WORD;
}
static KJ_ALWAYS_INLINE(ByteCount64 roundBitsUpToBytes(BitCount64 bits)) {
return (bits + 7 * BITS) / BITS_PER_BYTE;
return (bits + G(7) * BITS) / BITS_PER_BYTE;
}
#endif
static KJ_ALWAYS_INLINE(void zeroMemory(byte* ptr, ByteCount32 count)) {
memset(ptr, 0, unguard(count / BYTES));
}
static KJ_ALWAYS_INLINE(void zeroMemory(word* ptr, WordCountN<29> count)) {
memset(ptr, 0, unguard(count * BYTES_PER_WORD / BYTES));
}
static KJ_ALWAYS_INLINE(void zeroMemory(WirePointer* ptr, WirePointerCountN<29> count)) {
memset(ptr, 0, unguard(count * BYTES_PER_POINTER / BYTES));
}
static KJ_ALWAYS_INLINE(void copyMemory(byte* to, const byte* from, ByteCount32 count)) {
memcpy(to, from, unguard(count / BYTES));
}
static KJ_ALWAYS_INLINE(void copyMemory(word* to, const word* from, WordCountN<29> count)) {
memcpy(to, from, unguard(count * BYTES_PER_WORD / BYTES));
}
static KJ_ALWAYS_INLINE(void copyMemory(WirePointer* to, const WirePointer* from,
WirePointerCountN<29> count)) {
memcpy(to, from, unguard(count * BYTES_PER_POINTER / BYTES));
}
static KJ_ALWAYS_INLINE(bool boundsCheck(
......@@ -328,8 +383,8 @@ struct WireHelpers {
}
static KJ_ALWAYS_INLINE(word* allocate(
WirePointer*& ref, SegmentBuilder*& segment, CapTableBuilder* capTable, WordCount amount,
WirePointer::Kind kind, BuilderArena* orphanArena)) {
WirePointer*& ref, SegmentBuilder*& segment, CapTableBuilder* capTable,
SegmentWordCount amount, WirePointer::Kind kind, BuilderArena* orphanArena)) {
// Allocate space in the message for a new object, creating far pointers if necessary.
//
// * `ref` starts out being a reference to the pointer which shall be assigned to point at the
......@@ -353,7 +408,7 @@ struct WireHelpers {
if (orphanArena == nullptr) {
if (!ref->isNull()) zeroObject(segment, capTable, ref);
if (amount == 0 * WORDS && kind == WirePointer::STRUCT) {
if (amount == ZERO * WORDS && kind == WirePointer::STRUCT) {
// Note that the check for kind == WirePointer::STRUCT will hopefully cause this whole
// branch to be optimized away from all the call sites that are allocating non-structs.
ref->setKindAndTargetForEmptyStruct();
......@@ -368,7 +423,10 @@ struct WireHelpers {
// space to act as the landing pad for a far pointer.
WordCount amountPlusRef = amount + POINTER_SIZE_IN_WORDS;
auto allocation = segment->getArena()->allocate(amountPlusRef);
auto allocation = segment->getArena()->allocate(
assertMaxBits<SEGMENT_WORD_COUNT_BITS>(amountPlusRef, []() {
KJ_FAIL_REQUIRE("requested object size exceeds maximum segment size");
}));
segment = allocation.segment;
ptr = allocation.words;
......@@ -448,7 +506,7 @@ struct WireHelpers {
// Find the landing pad and check that it is within bounds.
const word* ptr = segment->getStartPtr() + ref->farPositionInSegment();
WordCount padWords = (1 + ref->isDoubleFar()) * POINTER_SIZE_IN_WORDS;
WordCount padWords = guarded(1 + ref->isDoubleFar()) * POINTER_SIZE_IN_WORDS;
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + padWords),
"Message contains out-of-bounds far pointer.") {
return nullptr;
......@@ -503,10 +561,10 @@ struct WireHelpers {
zeroObject(segment, capTable,
pad + 1, segment->getPtrUnchecked(pad->farPositionInSegment()));
}
memset(pad, 0, sizeof(WirePointer) * 2);
zeroMemory(pad, G(2) * POINTERS);
} else {
zeroObject(segment, capTable, pad);
memset(pad, 0, sizeof(WirePointer));
zeroMemory(pad, ONE * POINTERS);
}
}
break;
......@@ -534,11 +592,10 @@ struct WireHelpers {
case WirePointer::STRUCT: {
WirePointer* pointerSection =
reinterpret_cast<WirePointer*>(ptr + tag->structRef.dataSize.get());
uint count = tag->structRef.ptrCount.get() / POINTERS;
for (uint i = 0; i < count; i++) {
for (auto i: kj::zeroTo(tag->structRef.ptrCount.get())) {
zeroObject(segment, capTable, pointerSection + i);
}
memset(ptr, 0, tag->structRef.wordSize() * BYTES_PER_WORD / BYTES);
zeroMemory(ptr, tag->structRef.wordSize());
break;
}
case WirePointer::LIST: {
......@@ -550,18 +607,19 @@ struct WireHelpers {
case ElementSize::BYTE:
case ElementSize::TWO_BYTES:
case ElementSize::FOUR_BYTES:
case ElementSize::EIGHT_BYTES:
memset(ptr, 0,
roundBitsUpToWords(ElementCount64(tag->listRef.elementCount()) *
dataBitsPerElement(tag->listRef.elementSize()))
* BYTES_PER_WORD / BYTES);
case ElementSize::EIGHT_BYTES: {
zeroMemory(ptr, roundBitsUpToWords(
upgradeGuard<uint64_t>(tag->listRef.elementCount()) *
dataBitsPerElement(tag->listRef.elementSize())));
break;
}
case ElementSize::POINTER: {
uint count = tag->listRef.elementCount() / ELEMENTS;
for (uint i = 0; i < count; i++) {
zeroObject(segment, capTable, reinterpret_cast<WirePointer*>(ptr) + i);
WirePointer* typedPtr = reinterpret_cast<WirePointer*>(ptr);
auto count = tag->listRef.elementCount() * (ONE * POINTERS / ELEMENTS);
for (auto i: kj::zeroTo(count)) {
zeroObject(segment, capTable, typedPtr + i);
}
memset(ptr, 0, POINTER_SIZE_IN_WORDS * count * BYTES_PER_WORD / BYTES);
zeroMemory(typedPtr, count);
break;
}
case ElementSize::INLINE_COMPOSITE: {
......@@ -572,21 +630,25 @@ struct WireHelpers {
WordCount dataSize = elementTag->structRef.dataSize.get();
WirePointerCount pointerCount = elementTag->structRef.ptrCount.get();
uint count = elementTag->inlineCompositeListElementCount() / ELEMENTS;
auto count = elementTag->inlineCompositeListElementCount();
if (pointerCount > 0 * POINTERS) {
word* pos = ptr + POINTER_SIZE_IN_WORDS;
for (uint i = 0; i < count; i++) {
for (auto i KJ_UNUSED: kj::zeroTo(count)) {
pos += dataSize;
for (uint j = 0; j < pointerCount / POINTERS; j++) {
for (auto j KJ_UNUSED: kj::zeroTo(pointerCount)) {
zeroObject(segment, capTable, reinterpret_cast<WirePointer*>(pos));
pos += POINTER_SIZE_IN_WORDS;
}
}
}
memset(ptr, 0, (elementTag->structRef.wordSize() * count + POINTER_SIZE_IN_WORDS)
* BYTES_PER_WORD / BYTES);
auto wordsPerElement = elementTag->structRef.wordSize() / ELEMENTS;
zeroMemory(ptr, assertMaxBits<SEGMENT_WORD_COUNT_BITS>(POINTER_SIZE_IN_WORDS +
upgradeGuard<uint64_t>(count) * wordsPerElement, []() {
KJ_FAIL_ASSERT("encountered list pointer in builder which is too large to "
"possibly fit in a segment. Bug in builder code?");
}));
break;
}
}
......@@ -627,7 +689,7 @@ struct WireHelpers {
SegmentReader* segment, const WirePointer* ref, int nestingLimit) {
// Compute the total size of the object pointed to, not counting far pointer overhead.
MessageSizeCounts result = { 0 * WORDS, 0 };
MessageSizeCounts result = { ZERO * WORDS, 0 };
if (ref->isNull()) {
return result;
......@@ -646,12 +708,11 @@ struct WireHelpers {
"Message contained out-of-bounds struct pointer.") {
return result;
}
result.wordCount += ref->structRef.wordSize();
result.addWords(ref->structRef.wordSize());
const WirePointer* pointerSection =
reinterpret_cast<const WirePointer*>(ptr + ref->structRef.dataSize.get());
uint count = ref->structRef.ptrCount.get() / POINTERS;
for (uint i = 0; i < count; i++) {
for (auto i: kj::zeroTo(ref->structRef.ptrCount.get())) {
result += totalSize(segment, pointerSection + i, nestingLimit);
}
break;
......@@ -666,14 +727,14 @@ struct WireHelpers {
case ElementSize::TWO_BYTES:
case ElementSize::FOUR_BYTES:
case ElementSize::EIGHT_BYTES: {
WordCount64 totalWords = roundBitsUpToWords(
ElementCount64(ref->listRef.elementCount()) *
auto totalWords = roundBitsUpToWords(
upgradeGuard<uint64_t>(ref->listRef.elementCount()) *
dataBitsPerElement(ref->listRef.elementSize()));
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + totalWords),
"Message contained out-of-bounds list pointer.") {
return result;
}
result.wordCount += totalWords;
result.addWords(totalWords);
break;
}
case ElementSize::POINTER: {
......@@ -684,30 +745,31 @@ struct WireHelpers {
return result;
}
result.wordCount += count * WORDS_PER_POINTER;
result.addWords(count * WORDS_PER_POINTER);
for (uint i = 0; i < count / POINTERS; i++) {
for (auto i: kj::zeroTo(count)) {
result += totalSize(segment, reinterpret_cast<const WirePointer*>(ptr) + i,
nestingLimit);
}
break;
}
case ElementSize::INLINE_COMPOSITE: {
WordCount wordCount = ref->listRef.inlineCompositeWordCount();
auto wordCount = ref->listRef.inlineCompositeWordCount();
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + wordCount + POINTER_SIZE_IN_WORDS),
"Message contained out-of-bounds list pointer.") {
return result;
}
const WirePointer* elementTag = reinterpret_cast<const WirePointer*>(ptr);
ElementCount count = elementTag->inlineCompositeListElementCount();
auto count = elementTag->inlineCompositeListElementCount();
KJ_REQUIRE(elementTag->kind() == WirePointer::STRUCT,
"Don't know how to handle non-STRUCT inline composite.") {
return result;
}
auto actualSize = elementTag->structRef.wordSize() / ELEMENTS * ElementCount64(count);
auto actualSize = elementTag->structRef.wordSize() / ELEMENTS *
upgradeGuard<uint64_t>(count);
KJ_REQUIRE(actualSize <= wordCount,
"Struct list pointer's elements overran size.") {
return result;
......@@ -715,17 +777,17 @@ struct WireHelpers {
// We count the actual size rather than the claimed word count because that's what
// we'll end up with if we make a copy.
result.wordCount += actualSize + POINTER_SIZE_IN_WORDS;
result.addWords(wordCount + POINTER_SIZE_IN_WORDS);
WordCount dataSize = elementTag->structRef.dataSize.get();
WirePointerCount pointerCount = elementTag->structRef.ptrCount.get();
if (pointerCount > 0 * POINTERS) {
const word* pos = ptr + POINTER_SIZE_IN_WORDS;
for (uint i = 0; i < count / ELEMENTS; i++) {
for (auto i KJ_UNUSED: kj::zeroTo(count)) {
pos += dataSize;
for (uint j = 0; j < pointerCount / POINTERS; j++) {
for (auto j KJ_UNUSED: kj::zeroTo(pointerCount)) {
result += totalSize(segment, reinterpret_cast<const WirePointer*>(pos),
nestingLimit);
pos += POINTER_SIZE_IN_WORDS;
......@@ -760,13 +822,13 @@ struct WireHelpers {
static KJ_ALWAYS_INLINE(
void copyStruct(SegmentBuilder* segment, CapTableBuilder* capTable,
word* dst, const word* src,
WordCount dataSize, WirePointerCount pointerCount)) {
memcpy(dst, src, dataSize * BYTES_PER_WORD / BYTES);
StructDataWordCount dataSize, StructPointerCount pointerCount)) {
copyMemory(dst, src, dataSize);
const WirePointer* srcRefs = reinterpret_cast<const WirePointer*>(src + dataSize);
WirePointer* dstRefs = reinterpret_cast<WirePointer*>(dst + dataSize);
for (uint i = 0; i < pointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(pointerCount)) {
SegmentBuilder* subSegment = segment;
WirePointer* dstRef = dstRefs + i;
copyMessage(subSegment, capTable, dstRef, srcRefs + i);
......@@ -803,12 +865,12 @@ struct WireHelpers {
case ElementSize::TWO_BYTES:
case ElementSize::FOUR_BYTES:
case ElementSize::EIGHT_BYTES: {
WordCount wordCount = roundBitsUpToWords(
ElementCount64(src->listRef.elementCount()) *
auto wordCount = roundBitsUpToWords(
upgradeGuard<uint64_t>(src->listRef.elementCount()) *
dataBitsPerElement(src->listRef.elementSize()));
const word* srcPtr = src->target();
word* dstPtr = allocate(dst, segment, capTable, wordCount, WirePointer::LIST, nullptr);
memcpy(dstPtr, srcPtr, wordCount * BYTES_PER_WORD / BYTES);
copyMemory(dstPtr, srcPtr, wordCount);
dst->listRef.set(src->listRef.elementSize(), src->listRef.elementCount());
return dstPtr;
......@@ -818,11 +880,10 @@ struct WireHelpers {
const WirePointer* srcRefs = reinterpret_cast<const WirePointer*>(src->target());
WirePointer* dstRefs = reinterpret_cast<WirePointer*>(
allocate(dst, segment, capTable, src->listRef.elementCount() *
(1 * POINTERS / ELEMENTS) * WORDS_PER_POINTER,
(ONE * POINTERS / ELEMENTS) * WORDS_PER_POINTER,
WirePointer::LIST, nullptr));
uint n = src->listRef.elementCount() / ELEMENTS;
for (uint i = 0; i < n; i++) {
for (auto i: kj::zeroTo(src->listRef.elementCount() * (ONE * POINTERS / ELEMENTS))) {
SegmentBuilder* subSegment = segment;
WirePointer* dstRef = dstRefs + i;
copyMessage(subSegment, capTable, dstRef, srcRefs + i);
......@@ -835,7 +896,9 @@ struct WireHelpers {
case ElementSize::INLINE_COMPOSITE: {
const word* srcPtr = src->target();
word* dstPtr = allocate(dst, segment, capTable,
src->listRef.inlineCompositeWordCount() + POINTER_SIZE_IN_WORDS,
assertMaxBits<SEGMENT_WORD_COUNT_BITS>(
src->listRef.inlineCompositeWordCount() + POINTER_SIZE_IN_WORDS,
[]() { KJ_FAIL_ASSERT("list too big to fit in a segment"); }),
WirePointer::LIST, nullptr);
dst->listRef.setInlineComposite(src->listRef.inlineCompositeWordCount());
......@@ -849,8 +912,7 @@ struct WireHelpers {
KJ_ASSERT(srcTag->kind() == WirePointer::STRUCT,
"INLINE_COMPOSITE of lists is not yet supported.");
uint n = srcTag->inlineCompositeListElementCount() / ELEMENTS;
for (uint i = 0; i < n; i++) {
for (auto i KJ_UNUSED: kj::zeroTo(srcTag->inlineCompositeListElementCount())) {
copyStruct(segment, capTable, dstElement, srcElement,
srcTag->structRef.dataSize.get(), srcTag->structRef.ptrCount.get());
srcElement += srcTag->structRef.wordSize();
......@@ -917,10 +979,10 @@ struct WireHelpers {
// that it doesn't need to be a double-far.
WirePointer* landingPad =
reinterpret_cast<WirePointer*>(srcSegment->allocate(1 * WORDS));
reinterpret_cast<WirePointer*>(srcSegment->allocate(G(1) * WORDS));
if (landingPad == nullptr) {
// Darn, need a double-far.
auto allocation = srcSegment->getArena()->allocate(2 * WORDS);
auto allocation = srcSegment->getArena()->allocate(G(2) * WORDS);
SegmentBuilder* farSegment = allocation.segment;
landingPad = reinterpret_cast<WirePointer*>(allocation.words);
......@@ -988,8 +1050,8 @@ struct WireHelpers {
goto useDefault;
}
WordCount oldDataSize = oldRef->structRef.dataSize.get();
WirePointerCount oldPointerCount = oldRef->structRef.ptrCount.get();
auto oldDataSize = oldRef->structRef.dataSize.get();
auto oldPointerCount = oldRef->structRef.ptrCount.get();
WirePointer* oldPointerSection =
reinterpret_cast<WirePointer*>(oldPtr + oldDataSize);
......@@ -998,9 +1060,9 @@ struct WireHelpers {
// run with it and do bounds checks at access time, because how would we handle writes?
// Instead, we have to copy the struct to a new space now.
WordCount newDataSize = kj::max(oldDataSize, size.data);
WirePointerCount newPointerCount = kj::max(oldPointerCount, size.pointers);
WordCount totalSize = newDataSize + newPointerCount * WORDS_PER_POINTER;
auto newDataSize = kj::max(oldDataSize, size.data);
auto newPointerCount = kj::max(oldPointerCount, size.pointers);
auto totalSize = newDataSize + newPointerCount * WORDS_PER_POINTER;
// Don't let allocate() zero out the object just yet.
zeroPointerAndFars(segment, ref);
......@@ -1009,11 +1071,11 @@ struct WireHelpers {
ref->structRef.set(newDataSize, newPointerCount);
// Copy data section.
memcpy(ptr, oldPtr, oldDataSize * BYTES_PER_WORD / BYTES);
copyMemory(ptr, oldPtr, oldDataSize);
// Copy pointer section.
WirePointer* newPointerSection = reinterpret_cast<WirePointer*>(ptr + newDataSize);
for (uint i = 0; i < oldPointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(oldPointerCount)) {
transferPointer(segment, newPointerSection + i, oldSegment, oldPointerSection + i);
}
......@@ -1022,8 +1084,7 @@ struct WireHelpers {
// out as it may contain secrets that the caller intends to remove from the new copy.
// 2) Zeros will be deflated by packing, making this dead memory almost-free if it ever
// hits the wire.
memset(oldPtr, 0,
(oldDataSize + oldPointerCount * WORDS_PER_POINTER) * BYTES_PER_WORD / BYTES);
zeroMemory(oldPtr, oldDataSize + oldPointerCount * WORDS_PER_POINTER);
return StructBuilder(segment, capTable, ptr, newPointerSection, newDataSize * BITS_PER_WORD,
newPointerCount);
......@@ -1039,31 +1100,40 @@ struct WireHelpers {
KJ_DREQUIRE(elementSize != ElementSize::INLINE_COMPOSITE,
"Should have called initStructListPointer() instead.");
BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
WirePointerCount pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
auto checkedElementCount = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(elementCount,
[]() { KJ_FAIL_REQUIRE("tried to allocate list with too many elements"); });
auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto step = bitsPerElementIncludingPointers(elementSize);
KJ_DASSERT(step * ELEMENTS == (dataSize + pointerCount * BITS_PER_POINTER));
// Calculate size of the list.
WordCount wordCount = roundBitsUpToWords(ElementCount64(elementCount) * step);
auto wordCount = roundBitsUpToWords(upgradeGuard<uint64_t>(checkedElementCount) * step);
// Allocate the list.
word* ptr = allocate(ref, segment, capTable, wordCount, WirePointer::LIST, orphanArena);
// Initialize the pointer.
ref->listRef.set(elementSize, elementCount);
ref->listRef.set(elementSize, checkedElementCount);
// Build the ListBuilder.
return ListBuilder(segment, capTable, ptr, step, elementCount, dataSize,
pointerCount, elementSize);
return ListBuilder(segment, capTable, ptr, step, checkedElementCount,
dataSize, pointerCount, elementSize);
}
static KJ_ALWAYS_INLINE(ListBuilder initStructListPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
ElementCount elementCount, StructSize elementSize, BuilderArena* orphanArena = nullptr)) {
auto wordsPerElement = elementSize.total() / ELEMENTS;
auto checkedElementCount = assertMaxBits<LIST_ELEMENT_COUNT_BITS>(elementCount,
[]() { KJ_FAIL_REQUIRE("tried to allocate list with too many elements"); });
WordsPerElementN<17> wordsPerElement = elementSize.total() / ELEMENTS;
// Allocate the list, prefixed by a single WirePointer.
WordCount wordCount = elementCount * wordsPerElement;
auto wordCount = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
upgradeGuard<uint64_t>(checkedElementCount) * wordsPerElement,
[]() { KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size"); });
word* ptr = allocate(ref, segment, capTable, POINTER_SIZE_IN_WORDS + wordCount,
WirePointer::LIST, orphanArena);
......@@ -1073,12 +1143,12 @@ struct WireHelpers {
// Initialize the list tag.
reinterpret_cast<WirePointer*>(ptr)->setKindAndInlineCompositeListElementCount(
WirePointer::STRUCT, elementCount);
WirePointer::STRUCT, checkedElementCount);
reinterpret_cast<WirePointer*>(ptr)->structRef.set(elementSize);
ptr += POINTER_SIZE_IN_WORDS;
// Build the ListBuilder.
return ListBuilder(segment, capTable, ptr, wordsPerElement * BITS_PER_WORD, elementCount,
return ListBuilder(segment, capTable, ptr, wordsPerElement * BITS_PER_WORD, checkedElementCount,
elementSize.data * BITS_PER_WORD, elementSize.pointers,
ElementSize::INLINE_COMPOSITE);
}
......@@ -1136,8 +1206,8 @@ struct WireHelpers {
"INLINE_COMPOSITE list with non-STRUCT elements not supported.");
ptr += POINTER_SIZE_IN_WORDS;
WordCount dataSize = tag->structRef.dataSize.get();
WirePointerCount pointerCount = tag->structRef.ptrCount.get();
auto dataSize = tag->structRef.dataSize.get();
auto pointerCount = tag->structRef.ptrCount.get();
switch (elementSize) {
case ElementSize::VOID:
......@@ -1156,14 +1226,14 @@ struct WireHelpers {
case ElementSize::TWO_BYTES:
case ElementSize::FOUR_BYTES:
case ElementSize::EIGHT_BYTES:
KJ_REQUIRE(dataSize >= 1 * WORDS,
KJ_REQUIRE(dataSize >= ONE * WORDS,
"Existing list value is incompatible with expected type.") {
goto useDefault;
}
break;
case ElementSize::POINTER:
KJ_REQUIRE(pointerCount >= 1 * POINTERS,
KJ_REQUIRE(pointerCount >= ONE * POINTERS,
"Existing list value is incompatible with expected type.") {
goto useDefault;
}
......@@ -1182,8 +1252,8 @@ struct WireHelpers {
tag->inlineCompositeListElementCount(),
dataSize * BITS_PER_WORD, pointerCount, ElementSize::INLINE_COMPOSITE);
} else {
BitCount dataSize = dataBitsPerElement(oldSize) * ELEMENTS;
WirePointerCount pointerCount = pointersPerElement(oldSize) * ELEMENTS;
auto dataSize = dataBitsPerElement(oldSize) * ELEMENTS;
auto pointerCount = pointersPerElement(oldSize) * ELEMENTS;
if (elementSize == ElementSize::BIT) {
KJ_REQUIRE(oldSize == ElementSize::BIT,
......@@ -1257,8 +1327,8 @@ struct WireHelpers {
tag->structRef.dataSize.get() * BITS_PER_WORD,
tag->structRef.ptrCount.get(), ElementSize::INLINE_COMPOSITE);
} else {
BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
WirePointerCount pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
return ListBuilder(segment, capTable, ptr, step, ref->listRef.elementCount(),
......@@ -1310,10 +1380,11 @@ struct WireHelpers {
goto useDefault;
}
WordCount oldDataSize = oldTag->structRef.dataSize.get();
WirePointerCount oldPointerCount = oldTag->structRef.ptrCount.get();
auto oldDataSize = oldTag->structRef.dataSize.get();
auto oldPointerCount = oldTag->structRef.ptrCount.get();
auto oldStep = (oldDataSize + oldPointerCount * WORDS_PER_POINTER) / ELEMENTS;
ElementCount elementCount = oldTag->inlineCompositeListElementCount();
auto elementCount = oldTag->inlineCompositeListElementCount();
if (oldDataSize >= elementSize.data && oldPointerCount >= elementSize.pointers) {
// Old size is at least as large as we need. Ship it.
......@@ -1325,10 +1396,13 @@ struct WireHelpers {
// The structs in this list are smaller than expected, probably written using an older
// version of the protocol. We need to make a copy and expand them.
WordCount newDataSize = kj::max(oldDataSize, elementSize.data);
WirePointerCount newPointerCount = kj::max(oldPointerCount, elementSize.pointers);
auto newDataSize = kj::max(oldDataSize, elementSize.data);
auto newPointerCount = kj::max(oldPointerCount, elementSize.pointers);
auto newStep = (newDataSize + newPointerCount * WORDS_PER_POINTER) / ELEMENTS;
WordCount totalSize = newStep * elementCount;
auto totalSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
newStep * upgradeGuard<uint64_t>(elementCount),
[]() { KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size"); });
// Don't let allocate() zero out the object just yet.
zeroPointerAndFars(origSegment, origRef);
......@@ -1344,35 +1418,39 @@ struct WireHelpers {
word* src = oldPtr;
word* dst = newPtr;
for (uint i = 0; i < elementCount / ELEMENTS; i++) {
for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
// Copy data section.
memcpy(dst, src, oldDataSize * BYTES_PER_WORD / BYTES);
copyMemory(dst, src, oldDataSize);
// Copy pointer section.
WirePointer* newPointerSection = reinterpret_cast<WirePointer*>(dst + newDataSize);
WirePointer* oldPointerSection = reinterpret_cast<WirePointer*>(src + oldDataSize);
for (uint j = 0; j < oldPointerCount / POINTERS; j++) {
for (auto j: kj::zeroTo(oldPointerCount)) {
transferPointer(origSegment, newPointerSection + j, oldSegment, oldPointerSection + j);
}
dst += newStep * (1 * ELEMENTS);
src += oldStep * (1 * ELEMENTS);
dst += newStep * (ONE * ELEMENTS);
src += oldStep * (ONE * ELEMENTS);
}
auto oldSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
oldStep * upgradeGuard<uint64_t>(elementCount),
[]() { KJ_FAIL_ASSERT("old size overflows but new size doesn't?"); });
// Zero out old location. See explanation in getWritableStructPointer().
// Make sure to include the tag word.
memset(oldPtr - POINTER_SIZE_IN_WORDS, 0,
(POINTER_SIZE_IN_WORDS + oldStep * elementCount) * BYTES_PER_WORD / BYTES);
zeroMemory(oldPtr - POINTER_SIZE_IN_WORDS, oldSize + POINTER_SIZE_IN_WORDS);
return ListBuilder(origSegment, capTable, newPtr, newStep * BITS_PER_WORD, elementCount,
newDataSize * BITS_PER_WORD, newPointerCount, ElementSize::INLINE_COMPOSITE);
newDataSize * BITS_PER_WORD, newPointerCount,
ElementSize::INLINE_COMPOSITE);
} else {
// We're upgrading from a non-struct list.
BitCount oldDataSize = dataBitsPerElement(oldSize) * ELEMENTS;
WirePointerCount oldPointerCount = pointersPerElement(oldSize) * ELEMENTS;
auto oldDataSize = dataBitsPerElement(oldSize) * ELEMENTS;
auto oldPointerCount = pointersPerElement(oldSize) * ELEMENTS;
auto oldStep = (oldDataSize + oldPointerCount * BITS_PER_POINTER) / ELEMENTS;
ElementCount elementCount = oldRef->listRef.elementCount();
auto elementCount = oldRef->listRef.elementCount();
if (oldSize == ElementSize::VOID) {
// Nothing to copy, just allocate a new list.
......@@ -1386,18 +1464,20 @@ struct WireHelpers {
goto useDefault;
}
WordCount newDataSize = elementSize.data;
WirePointerCount newPointerCount = elementSize.pointers;
auto newDataSize = elementSize.data;
auto newPointerCount = elementSize.pointers;
if (oldSize == ElementSize::POINTER) {
newPointerCount = kj::max(newPointerCount, 1 * POINTERS);
newPointerCount = kj::max(newPointerCount, ONE * POINTERS);
} else {
// Old list contains data elements, so we need at least 1 word of data.
newDataSize = kj::max(newDataSize, 1 * WORDS);
newDataSize = kj::max(newDataSize, ONE * WORDS);
}
auto newStep = (newDataSize + newPointerCount * WORDS_PER_POINTER) / ELEMENTS;
WordCount totalWords = elementCount * newStep;
auto totalWords = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
newStep * upgradeGuard<uint64_t>(elementCount),
[]() {KJ_FAIL_REQUIRE("total size of struct list is larger than max segment size");});
// Don't let allocate() zero out the object just yet.
zeroPointerAndFars(origSegment, origRef);
......@@ -1414,24 +1494,29 @@ struct WireHelpers {
if (oldSize == ElementSize::POINTER) {
WirePointer* dst = reinterpret_cast<WirePointer*>(newPtr + newDataSize);
WirePointer* src = reinterpret_cast<WirePointer*>(oldPtr);
for (uint i = 0; i < elementCount / ELEMENTS; i++) {
for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
transferPointer(origSegment, dst, oldSegment, src);
dst += newStep / WORDS_PER_POINTER * (1 * ELEMENTS);
dst += newStep / WORDS_PER_POINTER * (ONE * ELEMENTS);
++src;
}
} else {
word* dst = newPtr;
char* src = reinterpret_cast<char*>(oldPtr);
ByteCount oldByteStep = oldDataSize / BITS_PER_BYTE;
for (uint i = 0; i < elementCount / ELEMENTS; i++) {
memcpy(dst, src, oldByteStep / BYTES);
src += oldByteStep / BYTES;
dst += newStep * (1 * ELEMENTS);
byte* dst = reinterpret_cast<byte*>(newPtr);
byte* src = reinterpret_cast<byte*>(oldPtr);
auto newByteStep = newStep * (ONE * ELEMENTS) * BYTES_PER_WORD;
auto oldByteStep = oldDataSize / BITS_PER_BYTE;
for (auto i KJ_UNUSED: kj::zeroTo(elementCount)) {
copyMemory(dst, src, oldByteStep);
src += oldByteStep;
dst += newByteStep;
}
}
auto oldSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
roundBitsUpToWords(oldStep * upgradeGuard<uint64_t>(elementCount)),
[]() { KJ_FAIL_ASSERT("old size overflows but new size doesn't?"); });
// Zero out old location. See explanation in getWritableStructPointer().
memset(oldPtr, 0, roundBitsUpToBytes(oldStep * elementCount) / BYTES);
zeroMemory(oldPtr, oldSize);
return ListBuilder(origSegment, capTable, newPtr, newStep * BITS_PER_WORD, elementCount,
newDataSize * BITS_PER_WORD, newPointerCount,
......@@ -1441,46 +1526,50 @@ struct WireHelpers {
}
static KJ_ALWAYS_INLINE(SegmentAnd<Text::Builder> initTextPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, ByteCount size,
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, TextSize size,
BuilderArena* orphanArena = nullptr)) {
// The byte list must include a NUL terminator.
ByteCount byteSize = size + 1 * BYTES;
auto byteSize = size + ONE * BYTES;
// Allocate the space.
word* ptr = allocate(
ref, segment, capTable, roundBytesUpToWords(byteSize), WirePointer::LIST, orphanArena);
// Initialize the pointer.
ref->listRef.set(ElementSize::BYTE, byteSize * (1 * ELEMENTS / BYTES));
ref->listRef.set(ElementSize::BYTE, byteSize * (ONE * ELEMENTS / BYTES));
// Build the Text::Builder. This will initialize the NUL terminator.
return { segment, Text::Builder(reinterpret_cast<char*>(ptr), size / BYTES) };
return { segment, Text::Builder(reinterpret_cast<char*>(ptr), unguard(size / BYTES)) };
}
static KJ_ALWAYS_INLINE(SegmentAnd<Text::Builder> setTextPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, Text::Reader value,
BuilderArena* orphanArena = nullptr)) {
auto allocation = initTextPointer(ref, segment, capTable, value.size() * BYTES, orphanArena);
TextSize size = assertMax<MAX_TEXT_SIZE>(guarded(value.size()),
[]() { KJ_FAIL_REQUIRE("text blob too big"); }) * BYTES;
auto allocation = initTextPointer(ref, segment, capTable, size, orphanArena);
memcpy(allocation.value.begin(), value.begin(), value.size());
return allocation;
}
static KJ_ALWAYS_INLINE(Text::Builder getWritableTextPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
const void* defaultValue, ByteCount defaultSize)) {
return getWritableTextPointer(ref, ref->target(), segment, capTable, defaultValue, defaultSize);
const void* defaultValue, TextSize defaultSize)) {
return getWritableTextPointer(ref, ref->target(), segment,capTable, defaultValue, defaultSize);
}
static KJ_ALWAYS_INLINE(Text::Builder getWritableTextPointer(
WirePointer* ref, word* refTarget, SegmentBuilder* segment, CapTableBuilder* capTable,
const void* defaultValue, ByteCount defaultSize)) {
const void* defaultValue, TextSize defaultSize)) {
if (ref->isNull()) {
useDefault:
if (defaultSize == 0 * BYTES) {
if (defaultSize == ZERO * BYTES) {
return nullptr;
} else {
Text::Builder builder = initTextPointer(ref, segment, capTable, defaultSize).value;
memcpy(builder.begin(), defaultValue, defaultSize / BYTES);
copyMemory(builder.asBytes().begin(), reinterpret_cast<const byte*>(defaultValue),
defaultSize);
return builder;
}
} else {
......@@ -1492,52 +1581,56 @@ struct WireHelpers {
KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
"Called getText{Field,Element}() but existing list pointer is not byte-sized.");
size_t size = ref->listRef.elementCount() / ELEMENTS;
KJ_REQUIRE(size > 0 && cptr[size-1] == '\0', "Text blob missing NUL terminator.") {
size_t size = unguard(subtractChecked(ref->listRef.elementCount() / ELEMENTS, ONE,
[]() { KJ_FAIL_REQUIRE("zero-size blob can't be text (need NUL terminator)"); }));
KJ_REQUIRE(cptr[size] == '\0', "Text blob missing NUL terminator.") {
goto useDefault;
}
return Text::Builder(cptr, size - 1);
return Text::Builder(cptr, size);
}
}
static KJ_ALWAYS_INLINE(SegmentAnd<Data::Builder> initDataPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, ByteCount size,
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, BlobSize size,
BuilderArena* orphanArena = nullptr)) {
// Allocate the space.
word* ptr = allocate(ref, segment, capTable, roundBytesUpToWords(size),
WirePointer::LIST, orphanArena);
// Initialize the pointer.
ref->listRef.set(ElementSize::BYTE, size * (1 * ELEMENTS / BYTES));
ref->listRef.set(ElementSize::BYTE, size * (ONE * ELEMENTS / BYTES));
// Build the Data::Builder.
return { segment, Data::Builder(reinterpret_cast<byte*>(ptr), size / BYTES) };
return { segment, Data::Builder(reinterpret_cast<byte*>(ptr), unguard(size / BYTES)) };
}
static KJ_ALWAYS_INLINE(SegmentAnd<Data::Builder> setDataPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable, Data::Reader value,
BuilderArena* orphanArena = nullptr)) {
auto allocation = initDataPointer(ref, segment, capTable, value.size() * BYTES, orphanArena);
BlobSize size = assertMaxBits<BLOB_SIZE_BITS>(guarded(value.size()),
[]() { KJ_FAIL_REQUIRE("text blob too big"); }) * BYTES;
auto allocation = initDataPointer(ref, segment, capTable, size, orphanArena);
memcpy(allocation.value.begin(), value.begin(), value.size());
return allocation;
}
static KJ_ALWAYS_INLINE(Data::Builder getWritableDataPointer(
WirePointer* ref, SegmentBuilder* segment, CapTableBuilder* capTable,
const void* defaultValue, ByteCount defaultSize)) {
const void* defaultValue, BlobSize defaultSize)) {
return getWritableDataPointer(ref, ref->target(), segment, capTable, defaultValue, defaultSize);
}
static KJ_ALWAYS_INLINE(Data::Builder getWritableDataPointer(
WirePointer* ref, word* refTarget, SegmentBuilder* segment, CapTableBuilder* capTable,
const void* defaultValue, ByteCount defaultSize)) {
const void* defaultValue, BlobSize defaultSize)) {
if (ref->isNull()) {
if (defaultSize == 0 * BYTES) {
if (defaultSize == ZERO * BYTES) {
return nullptr;
} else {
Data::Builder builder = initDataPointer(ref, segment, capTable, defaultSize).value;
memcpy(builder.begin(), defaultValue, defaultSize / BYTES);
copyMemory(builder.begin(), reinterpret_cast<const byte*>(defaultValue), defaultSize);
return builder;
}
} else {
......@@ -1548,15 +1641,17 @@ struct WireHelpers {
KJ_REQUIRE(ref->listRef.elementSize() == ElementSize::BYTE,
"Called getData{Field,Element}() but existing list pointer is not byte-sized.");
return Data::Builder(reinterpret_cast<byte*>(ptr), ref->listRef.elementCount() / ELEMENTS);
return Data::Builder(reinterpret_cast<byte*>(ptr),
unguard(ref->listRef.elementCount() / ELEMENTS));
}
}
static SegmentAnd<word*> setStructPointer(
SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, StructReader value,
BuilderArena* orphanArena = nullptr, bool canonical = false) {
ByteCount dataSize = roundBitsUpToBytes(value.dataSize);
WirePointerCount ptrCount = value.pointerCount;
// TODO(now): Function may have been damaged in merge conflict resolution.
auto dataSize = roundBitsUpToBytes(value.dataSize);
auto ptrCount = value.pointerCount;
if (canonical) {
// StructReaders should not have bitwidths other than 1, but let's be safe
......@@ -1604,17 +1699,19 @@ struct WireHelpers {
word* ptr = allocate(ref, segment, capTable, totalSize, WirePointer::STRUCT, orphanArena);
ref->structRef.set(dataWords, ptrCount);
if (value.dataSize == 1 * BITS) {
if (value.dataSize == ONE * BITS) {
// Data size could be made 0 by truncation
if (dataSize != 0 * BYTES) {
*reinterpret_cast<char*>(ptr) = value.getDataField<bool>(0 * ELEMENTS);
if (dataSize != ZERO * BYTES) {
*reinterpret_cast<char*>(ptr) = value.getDataField<bool>(ZERO * ELEMENTS);
}
} else {
memcpy(ptr, value.data, dataSize / BYTES);
copyMemory(reinterpret_cast<byte*>(ptr),
reinterpret_cast<const byte*>(value.data),
dataSize);
}
WirePointer* pointerSection = reinterpret_cast<WirePointer*>(ptr + dataWords);
for (uint i = 0; i < ptrCount / POINTERS; i++) {
for (auto i: kj::zeroTo(ptrCount)) {
copyPointer(segment, capTable, pointerSection + i,
value.segment, value.capTable, value.pointers + i,
value.nestingLimit, nullptr, canonical);
......@@ -1641,7 +1738,10 @@ struct WireHelpers {
static SegmentAnd<word*> setListPointer(
SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, ListReader value,
BuilderArena* orphanArena = nullptr, bool canonical = false) {
WordCount totalSize = roundBitsUpToWords(value.elementCount * value.step);
// TODO(now): Function may have been damaged in merge conflict resolution.
auto totalSize = assertMax<kj::maxValueForBits<SEGMENT_WORD_COUNT_BITS>() - 1>(
roundBitsUpToWords(upgradeGuard<uint64_t>(value.elementCount) * value.step),
[]() { KJ_FAIL_ASSERT("encountered impossibly long struct list ListReader"); });
if (value.elementSize != ElementSize::INLINE_COMPOSITE) {
// List of non-structs.
......@@ -1650,7 +1750,7 @@ struct WireHelpers {
if (value.elementSize == ElementSize::POINTER) {
// List of pointers.
ref->listRef.set(ElementSize::POINTER, value.elementCount);
for (uint i = 0; i < value.elementCount / ELEMENTS; i++) {
for (auto i: zeroTo(value.elementCount * (ONE * POINTERS / ELEMENTS))) {
copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(ptr) + i,
value.segment, value.capTable,
reinterpret_cast<const WirePointer*>(value.ptr) + i,
......@@ -1659,14 +1759,12 @@ struct WireHelpers {
} else {
// List of data.
ref->listRef.set(value.elementSize, value.elementCount);
memcpy(ptr, value.ptr, totalSize * BYTES_PER_WORD / BYTES);
copyMemory(ptr, reinterpret_cast<const word*>(value.ptr), totalSize);
}
return { segment, ptr };
} else {
// List of structs.
KJ_DASSERT(value.structDataSize % BITS_PER_WORD == 0 * BITS);
WordCount declDataSize = value.structDataSize / BITS_PER_WORD;
WirePointerCount declPointerCount = value.structPointerCount;
......@@ -1711,6 +1809,7 @@ struct WireHelpers {
ptrCount = declPointerCount;
}
KJ_DASSERT(value.structDataSize % BITS_PER_WORD == 0 * BITS);
word* ptr = allocate(ref, segment, capTable, totalSize + POINTER_SIZE_IN_WORDS,
WirePointer::LIST, orphanArena);
ref->listRef.setInlineComposite(totalSize);
......@@ -1721,12 +1820,12 @@ struct WireHelpers {
word* dst = ptr + POINTER_SIZE_IN_WORDS;
const word* src = reinterpret_cast<const word*>(value.ptr);
for (uint i = 0; i < value.elementCount / ELEMENTS; i++) {
memcpy(dst, src, dataSize * BYTES_PER_WORD / BYTES);
for (auto i KJ_UNUSED: kj::zeroTo(value.elementCount)) {
copyMemory(dst, src, dataSize);
dst += dataSize;
src += declDataSize;
for (uint j = 0; j < ptrCount / POINTERS; j++) {
for (auto j KJ_UNUSED: kj::zeroTo(ptrCount)) {
copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(dst),
value.segment, value.capTable, reinterpret_cast<const WirePointer*>(src),
value.nestingLimit, nullptr, canonical);
......@@ -1802,7 +1901,7 @@ struct WireHelpers {
}
if (elementSize == ElementSize::INLINE_COMPOSITE) {
WordCount wordCount = src->listRef.inlineCompositeWordCount();
auto wordCount = src->listRef.inlineCompositeWordCount();
const WirePointer* tag = reinterpret_cast<const WirePointer*>(ptr);
ptr += POINTER_SIZE_IN_WORDS;
......@@ -1816,10 +1915,10 @@ struct WireHelpers {
goto useDefault;
}
ElementCount elementCount = tag->inlineCompositeListElementCount();
auto elementCount = tag->inlineCompositeListElementCount();
auto wordsPerElement = tag->structRef.wordSize() / ELEMENTS;
KJ_REQUIRE(wordsPerElement * ElementCount64(elementCount) <= wordCount,
KJ_REQUIRE(wordsPerElement * upgradeGuard<uint64_t>(elementCount) <= wordCount,
"INLINE_COMPOSITE list's elements overrun its word count.") {
goto useDefault;
}
......@@ -1841,11 +1940,11 @@ struct WireHelpers {
nestingLimit - 1),
orphanArena, canonical);
} else {
BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
WirePointerCount pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
auto pointerCount = pointersPerElement(elementSize) * ELEMENTS;
auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
ElementCount elementCount = src->listRef.elementCount();
WordCount64 wordCount = roundBitsUpToWords(ElementCount64(elementCount) * step);
auto elementCount = src->listRef.elementCount();
auto wordCount = roundBitsUpToWords(upgradeGuard<uint64_t>(elementCount) * step);
KJ_REQUIRE(boundsCheck(srcSegment, ptr, ptr + wordCount),
"Message contains out-of-bounds list pointer.") {
......@@ -2081,15 +2180,7 @@ struct WireHelpers {
ElementSize elementSize = ref->listRef.elementSize();
if (elementSize == ElementSize::INLINE_COMPOSITE) {
#if _MSC_VER
// TODO(msvc): MSVC thinks decltype(WORDS/ELEMENTS) is a const type. /eyeroll
uint wordsPerElement;
#else
decltype(WORDS/ELEMENTS) wordsPerElement;
#endif
ElementCount size;
WordCount wordCount = ref->listRef.inlineCompositeWordCount();
auto wordCount = ref->listRef.inlineCompositeWordCount();
// An INLINE_COMPOSITE list points to a tag, which is formatted like a pointer.
const WirePointer* tag = reinterpret_cast<const WirePointer*>(ptr);
......@@ -2105,18 +2196,18 @@ struct WireHelpers {
goto useDefault;
}
size = tag->inlineCompositeListElementCount();
wordsPerElement = tag->structRef.wordSize() / ELEMENTS;
auto size = tag->inlineCompositeListElementCount();
auto wordsPerElement = tag->structRef.wordSize() / ELEMENTS;
KJ_REQUIRE(ElementCount64(size) * wordsPerElement <= wordCount,
KJ_REQUIRE(upgradeGuard<uint64_t>(size) * wordsPerElement <= wordCount,
"INLINE_COMPOSITE list's elements overrun its word count.") {
goto useDefault;
}
if (wordsPerElement * (1 * ELEMENTS) == 0 * WORDS) {
if (wordsPerElement * (ONE * ELEMENTS) == ZERO * WORDS) {
// Watch out for lists of zero-sized structs, which can claim to be arbitrarily large
// without having sent actual data.
KJ_REQUIRE(amplifiedRead(segment, size * (1 * WORDS / ELEMENTS)),
KJ_REQUIRE(amplifiedRead(segment, size * (ONE * WORDS / ELEMENTS)),
"Message contains amplified list pointer.") {
goto useDefault;
}
......@@ -2145,7 +2236,7 @@ struct WireHelpers {
case ElementSize::TWO_BYTES:
case ElementSize::FOUR_BYTES:
case ElementSize::EIGHT_BYTES:
KJ_REQUIRE(tag->structRef.dataSize.get() > 0 * WORDS,
KJ_REQUIRE(tag->structRef.dataSize.get() > ZERO * WORDS,
"Expected a primitive list, but got a list of pointer-only structs.") {
goto useDefault;
}
......@@ -2156,7 +2247,7 @@ struct WireHelpers {
// in the struct is the pointer we were looking for, we want to munge the pointer to
// point at the first element's pointer section.
ptr += tag->structRef.dataSize.get();
KJ_REQUIRE(tag->structRef.ptrCount.get() > 0 * POINTERS,
KJ_REQUIRE(tag->structRef.ptrCount.get() > ZERO * POINTERS,
"Expected a pointer list, but got a list of data-only structs.") {
goto useDefault;
}
......@@ -2176,22 +2267,21 @@ struct WireHelpers {
} else {
// This is a primitive or pointer list, but all such lists can also be interpreted as struct
// lists. We need to compute the data size and pointer count for such structs.
BitCount dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
WirePointerCount pointerCount =
pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
ElementCount elementCount = ref->listRef.elementCount();
auto dataSize = dataBitsPerElement(ref->listRef.elementSize()) * ELEMENTS;
auto pointerCount = pointersPerElement(ref->listRef.elementSize()) * ELEMENTS;
auto elementCount = ref->listRef.elementCount();
auto step = (dataSize + pointerCount * BITS_PER_POINTER) / ELEMENTS;
WordCount wordCount = roundBitsUpToWords(ElementCount64(elementCount) * step);
auto wordCount = roundBitsUpToWords(upgradeGuard<uint64_t>(elementCount) * step);
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + wordCount),
"Message contains out-of-bounds list pointer.") {
"Message contains out-of-bounds list pointer.") {
goto useDefault;
}
if (elementSize == ElementSize::VOID) {
// Watch out for lists of void, which can claim to be arbitrarily large without having sent
// actual data.
KJ_REQUIRE(amplifiedRead(segment, elementCount * (1 * WORDS / ELEMENTS)),
KJ_REQUIRE(amplifiedRead(segment, elementCount * (ONE * WORDS / ELEMENTS)),
"Message contains amplified list pointer.") {
goto useDefault;
}
......@@ -2243,7 +2333,8 @@ struct WireHelpers {
if (ref->isNull()) {
useDefault:
if (defaultValue == nullptr) defaultValue = "";
return Text::Reader(reinterpret_cast<const char*>(defaultValue), defaultSize / BYTES);
return Text::Reader(reinterpret_cast<const char*>(defaultValue),
unguard(defaultSize / BYTES));
} else {
const word* ptr = followFars(ref, refTarget, segment);
......@@ -2252,7 +2343,7 @@ struct WireHelpers {
goto useDefault;
}
uint size = ref->listRef.elementCount() / ELEMENTS;
auto size = ref->listRef.elementCount() * (ONE * BYTES / ELEMENTS);
KJ_REQUIRE(ref->kind() == WirePointer::LIST,
"Message contains non-list pointer where text was expected.") {
......@@ -2264,39 +2355,39 @@ struct WireHelpers {
goto useDefault;
}
KJ_REQUIRE(boundsCheck(segment, ptr, ptr +
roundBytesUpToWords(ref->listRef.elementCount() * (1 * BYTES / ELEMENTS))),
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + roundBytesUpToWords(size)),
"Message contained out-of-bounds text pointer.") {
goto useDefault;
}
KJ_REQUIRE(size > 0, "Message contains text that is not NUL-terminated.") {
KJ_REQUIRE(size > ZERO * BYTES, "Message contains text that is not NUL-terminated.") {
goto useDefault;
}
const char* cptr = reinterpret_cast<const char*>(ptr);
--size; // NUL terminator
uint unguardedSize = unguard(size / BYTES) - 1;
KJ_REQUIRE(cptr[size] == '\0', "Message contains text that is not NUL-terminated.") {
KJ_REQUIRE(cptr[unguardedSize] == '\0', "Message contains text that is not NUL-terminated.") {
goto useDefault;
}
return Text::Reader(cptr, size);
return Text::Reader(cptr, unguardedSize);
}
}
static KJ_ALWAYS_INLINE(Data::Reader readDataPointer(
SegmentReader* segment, const WirePointer* ref,
const void* defaultValue, ByteCount defaultSize)) {
const void* defaultValue, BlobSize defaultSize)) {
return readDataPointer(segment, ref, ref->target(), defaultValue, defaultSize);
}
static KJ_ALWAYS_INLINE(Data::Reader readDataPointer(
SegmentReader* segment, const WirePointer* ref, const word* refTarget,
const void* defaultValue, ByteCount defaultSize)) {
const void* defaultValue, BlobSize defaultSize)) {
if (ref->isNull()) {
useDefault:
return Data::Reader(reinterpret_cast<const byte*>(defaultValue), defaultSize / BYTES);
return Data::Reader(reinterpret_cast<const byte*>(defaultValue),
unguard(defaultSize / BYTES));
} else {
const word* ptr = followFars(ref, refTarget, segment);
......@@ -2305,7 +2396,7 @@ struct WireHelpers {
goto useDefault;
}
uint size = ref->listRef.elementCount() / ELEMENTS;
auto size = ref->listRef.elementCount() * (ONE * BYTES / ELEMENTS);
KJ_REQUIRE(ref->kind() == WirePointer::LIST,
"Message contains non-list pointer where data was expected.") {
......@@ -2317,13 +2408,12 @@ struct WireHelpers {
goto useDefault;
}
KJ_REQUIRE(boundsCheck(segment, ptr, ptr +
roundBytesUpToWords(ref->listRef.elementCount() * (1 * BYTES / ELEMENTS))),
KJ_REQUIRE(boundsCheck(segment, ptr, ptr + roundBytesUpToWords(size)),
"Message contained out-of-bounds data pointer.") {
goto useDefault;
}
return Data::Reader(reinterpret_cast<const byte*>(ptr), size);
return Data::Reader(reinterpret_cast<const byte*>(ptr), unguard(size / BYTES));
}
}
};
......@@ -2362,7 +2452,8 @@ ListBuilder PointerBuilder::getListAnySize(const word* defaultValue) {
template <>
Text::Builder PointerBuilder::initBlob<Text>(ByteCount size) {
return WireHelpers::initTextPointer(pointer, segment, capTable, size).value;
return WireHelpers::initTextPointer(pointer, segment, capTable,
assertMax<MAX_TEXT_SIZE>(size, ThrowOverflow())).value;
}
template <>
void PointerBuilder::setBlob<Text>(Text::Reader value) {
......@@ -2370,12 +2461,14 @@ void PointerBuilder::setBlob<Text>(Text::Reader value) {
}
template <>
Text::Builder PointerBuilder::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) {
return WireHelpers::getWritableTextPointer(pointer, segment, capTable, defaultValue, defaultSize);
return WireHelpers::getWritableTextPointer(pointer, segment, capTable, defaultValue,
assertMax<MAX_TEXT_SIZE>(defaultSize, ThrowOverflow()));
}
template <>
Data::Builder PointerBuilder::initBlob<Data>(ByteCount size) {
return WireHelpers::initDataPointer(pointer, segment, capTable, size).value;
return WireHelpers::initDataPointer(pointer, segment, capTable,
assertMaxBits<BLOB_SIZE_BITS>(size, ThrowOverflow())).value;
}
template <>
void PointerBuilder::setBlob<Data>(Data::Reader value) {
......@@ -2383,7 +2476,8 @@ void PointerBuilder::setBlob<Data>(Data::Reader value) {
}
template <>
Data::Builder PointerBuilder::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) {
return WireHelpers::getWritableDataPointer(pointer, segment, capTable, defaultValue, defaultSize);
return WireHelpers::getWritableDataPointer(pointer, segment, capTable, defaultValue,
assertMaxBits<BLOB_SIZE_BITS>(defaultSize, ThrowOverflow()));
}
void PointerBuilder::setStruct(const StructReader& value, bool canonical) {
......@@ -2521,7 +2615,8 @@ Text::Reader PointerReader::getBlob<Text>(const void* defaultValue, ByteCount de
template <>
Data::Reader PointerReader::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) const {
const WirePointer* ref = pointer == nullptr ? &zero.pointer : pointer;
return WireHelpers::readDataPointer(segment, ref, defaultValue, defaultSize);
return WireHelpers::readDataPointer(segment, ref, defaultValue,
assertMaxBits<BLOB_SIZE_BITS>(defaultSize, ThrowOverflow()));
}
#if !CAPNP_LITE
......@@ -2537,7 +2632,7 @@ const word* PointerReader::getUnchecked() const {
}
MessageSizeCounts PointerReader::targetSize() const {
return pointer == nullptr ? MessageSizeCounts { 0 * WORDS, 0 }
return pointer == nullptr ? MessageSizeCounts { ZERO * WORDS, 0 }
: WireHelpers::totalSize(segment, pointer, nestingLimit);
}
......@@ -2615,89 +2710,96 @@ bool PointerReader::isCanonical(const word **readHead) {
// StructBuilder
void StructBuilder::clearAll() {
if (dataSize == 1 * BITS) {
setDataField<bool>(1 * ELEMENTS, false);
if (dataSize == ONE * BITS) {
setDataField<bool>(ONE * ELEMENTS, false);
} else {
memset(data, 0, dataSize / BITS_PER_BYTE / BYTES);
WireHelpers::zeroMemory(reinterpret_cast<byte*>(data), dataSize / BITS_PER_BYTE);
}
for (uint i = 0; i < pointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(pointerCount)) {
WireHelpers::zeroObject(segment, capTable, pointers + i);
}
memset(pointers, 0, pointerCount * BYTES_PER_POINTER / BYTES);
WireHelpers::zeroMemory(pointers, pointerCount);
}
void StructBuilder::transferContentFrom(StructBuilder other) {
// Determine the amount of data the builders have in common.
BitCount sharedDataSize = kj::min(dataSize, other.dataSize);
auto sharedDataSize = kj::min(dataSize, other.dataSize);
if (dataSize > sharedDataSize) {
// Since the target is larger than the source, make sure to zero out the extra bits that the
// source doesn't have.
if (dataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, false);
if (dataSize == ONE * BITS) {
setDataField<bool>(ZERO * ELEMENTS, false);
} else {
byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE / BYTES;
memset(unshared, 0, (dataSize - sharedDataSize) / BITS_PER_BYTE / BYTES);
byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE;
// Note: this subtraction can't fail due to the if() above
WireHelpers::zeroMemory(unshared,
subtractChecked(dataSize, sharedDataSize, []() {}) / BITS_PER_BYTE);
}
}
// Copy over the shared part.
if (sharedDataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, other.getDataField<bool>(0 * ELEMENTS));
if (sharedDataSize == ONE * BITS) {
setDataField<bool>(ZERO * ELEMENTS, other.getDataField<bool>(ZERO * ELEMENTS));
} else {
memcpy(data, other.data, sharedDataSize / BITS_PER_BYTE / BYTES);
WireHelpers::copyMemory(reinterpret_cast<byte*>(data),
reinterpret_cast<byte*>(other.data),
sharedDataSize / BITS_PER_BYTE);
}
// Zero out all pointers in the target.
for (uint i = 0; i < pointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(pointerCount)) {
WireHelpers::zeroObject(segment, capTable, pointers + i);
}
memset(pointers, 0, pointerCount * BYTES_PER_POINTER / BYTES);
WireHelpers::zeroMemory(pointers, pointerCount);
// Transfer the pointers.
WirePointerCount sharedPointerCount = kj::min(pointerCount, other.pointerCount);
for (uint i = 0; i < sharedPointerCount / POINTERS; i++) {
auto sharedPointerCount = kj::min(pointerCount, other.pointerCount);
for (auto i: kj::zeroTo(sharedPointerCount)) {
WireHelpers::transferPointer(segment, pointers + i, other.segment, other.pointers + i);
}
// Zero out the pointers that were transferred in the source because it no longer has ownership.
// If the source had any extra pointers that the destination didn't have space for, we
// intentionally leave them be, so that they'll be cleaned up later.
memset(other.pointers, 0, sharedPointerCount * BYTES_PER_POINTER / BYTES);
WireHelpers::zeroMemory(other.pointers, sharedPointerCount);
}
void StructBuilder::copyContentFrom(StructReader other) {
// Determine the amount of data the builders have in common.
BitCount sharedDataSize = kj::min(dataSize, other.dataSize);
auto sharedDataSize = kj::min(dataSize, other.dataSize);
if (dataSize > sharedDataSize) {
// Since the target is larger than the source, make sure to zero out the extra bits that the
// source doesn't have.
if (dataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, false);
if (dataSize == ONE * BITS) {
setDataField<bool>(ZERO * ELEMENTS, false);
} else {
byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE / BYTES;
memset(unshared, 0, (dataSize - sharedDataSize) / BITS_PER_BYTE / BYTES);
byte* unshared = reinterpret_cast<byte*>(data) + sharedDataSize / BITS_PER_BYTE;
WireHelpers::zeroMemory(unshared,
subtractChecked(dataSize, sharedDataSize, []() {}) / BITS_PER_BYTE);
}
}
// Copy over the shared part.
if (sharedDataSize == 1 * BITS) {
setDataField<bool>(0 * ELEMENTS, other.getDataField<bool>(0 * ELEMENTS));
if (sharedDataSize == ONE * BITS) {
setDataField<bool>(ZERO * ELEMENTS, other.getDataField<bool>(ZERO * ELEMENTS));
} else {
memcpy(data, other.data, sharedDataSize / BITS_PER_BYTE / BYTES);
WireHelpers::copyMemory(reinterpret_cast<byte*>(data),
reinterpret_cast<const byte*>(other.data),
sharedDataSize / BITS_PER_BYTE);
}
// Zero out all pointers in the target.
for (uint i = 0; i < pointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(pointerCount)) {
WireHelpers::zeroObject(segment, capTable, pointers + i);
}
memset(pointers, 0, pointerCount * BYTES_PER_POINTER / BYTES);
WireHelpers::zeroMemory(pointers, pointerCount);
// Copy the pointers.
WirePointerCount sharedPointerCount = kj::min(pointerCount, other.pointerCount);
for (uint i = 0; i < sharedPointerCount / POINTERS; i++) {
auto sharedPointerCount = kj::min(pointerCount, other.pointerCount);
for (auto i: kj::zeroTo(sharedPointerCount)) {
WireHelpers::copyPointer(segment, capTable, pointers + i,
other.segment, other.capTable, other.pointers + i, other.nestingLimit);
}
......@@ -2729,7 +2831,7 @@ MessageSizeCounts StructReader::totalSize() const {
MessageSizeCounts result = {
WireHelpers::roundBitsUpToWords(dataSize) + pointerCount * WORDS_PER_POINTER, 0 };
for (uint i = 0; i < pointerCount / POINTERS; i++) {
for (auto i: kj::zeroTo(pointerCount)) {
result += WireHelpers::totalSize(segment, pointers + i, nestingLimit);
}
......@@ -2812,12 +2914,12 @@ bool StructReader::isCanonical(const word **readHead,
// ListBuilder
Text::Builder ListBuilder::asText() {
KJ_REQUIRE(structDataSize == 8 * BITS && structPointerCount == 0 * POINTERS,
KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
"Expected Text, got list of non-bytes.") {
return Text::Builder();
}
size_t size = elementCount / ELEMENTS;
size_t size = unguard(elementCount / ELEMENTS);
KJ_REQUIRE(size > 0, "Message contains text that is not NUL-terminated.") {
return Text::Builder();
......@@ -2834,18 +2936,18 @@ Text::Builder ListBuilder::asText() {
}
Data::Builder ListBuilder::asData() {
KJ_REQUIRE(structDataSize == 8 * BITS && structPointerCount == 0 * POINTERS,
KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
"Expected Text, got list of non-bytes.") {
return Data::Builder();
}
return Data::Builder(reinterpret_cast<byte*>(ptr), elementCount / ELEMENTS);
return Data::Builder(reinterpret_cast<byte*>(ptr), unguard(elementCount / ELEMENTS));
}
StructBuilder ListBuilder::getStructElement(ElementCount index) {
BitCount64 indexBit = ElementCount64(index) * step;
auto indexBit = upgradeGuard<uint64_t>(index) * step;
byte* structData = ptr + indexBit / BITS_PER_BYTE;
KJ_DASSERT(indexBit % BITS_PER_BYTE == 0 * BITS);
KJ_DASSERT(indexBit % BITS_PER_BYTE == ZERO * BITS);
return StructBuilder(segment, capTable, structData,
reinterpret_cast<WirePointer*>(structData + structDataSize / BITS_PER_BYTE),
structDataSize, structPointerCount);
......@@ -2874,12 +2976,12 @@ ListBuilder ListBuilder::imbue(CapTableBuilder* capTable) {
// ListReader
Text::Reader ListReader::asText() {
KJ_REQUIRE(structDataSize == 8 * BITS && structPointerCount == 0 * POINTERS,
KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
"Expected Text, got list of non-bytes.") {
return Text::Reader();
}
size_t size = elementCount / ELEMENTS;
size_t size = unguard(elementCount / ELEMENTS);
KJ_REQUIRE(size > 0, "Message contains text that is not NUL-terminated.") {
return Text::Reader();
......@@ -2896,12 +2998,12 @@ Text::Reader ListReader::asText() {
}
Data::Reader ListReader::asData() {
KJ_REQUIRE(structDataSize == 8 * BITS && structPointerCount == 0 * POINTERS,
KJ_REQUIRE(structDataSize == G(8) * BITS && structPointerCount == ZERO * POINTERS,
"Expected Text, got list of non-bytes.") {
return Data::Reader();
}
return Data::Reader(reinterpret_cast<const byte*>(ptr), elementCount / ELEMENTS);
return Data::Reader(reinterpret_cast<const byte*>(ptr), unguard(elementCount / ELEMENTS));
}
kj::ArrayPtr<const byte> ListReader::asRawBytes() {
......@@ -2920,17 +3022,17 @@ StructReader ListReader::getStructElement(ElementCount index) const {
return StructReader();
}
BitCount64 indexBit = ElementCount64(index) * step;
auto indexBit = upgradeGuard<uint64_t>(index) * step;
const byte* structData = ptr + indexBit / BITS_PER_BYTE;
const WirePointer* structPointers =
reinterpret_cast<const WirePointer*>(structData + structDataSize / BITS_PER_BYTE);
// This check should pass if there are no bugs in the list pointer validation code.
KJ_DASSERT(structPointerCount == 0 * POINTERS ||
KJ_DASSERT(structPointerCount == ZERO * POINTERS ||
(uintptr_t)structPointers % sizeof(void*) == 0,
"Pointer section of struct list element not aligned.");
KJ_DASSERT(indexBit % BITS_PER_BYTE == 0 * BITS);
KJ_DASSERT(indexBit % BITS_PER_BYTE == ZERO * BITS);
return StructReader(
segment, capTable, structData, structPointers,
structDataSize, structPointerCount,
......@@ -3062,7 +3164,8 @@ OrphanBuilder OrphanBuilder::initStructList(
OrphanBuilder OrphanBuilder::initText(
BuilderArena* arena, CapTableBuilder* capTable, ByteCount size) {
OrphanBuilder result;
auto allocation = WireHelpers::initTextPointer(result.tagAsPtr(), nullptr, capTable, size, arena);
auto allocation = WireHelpers::initTextPointer(result.tagAsPtr(), nullptr, capTable,
assertMax<MAX_TEXT_SIZE>(size, ThrowOverflow()), arena);
result.segment = allocation.segment;
result.capTable = capTable;
result.location = reinterpret_cast<word*>(allocation.value.begin());
......@@ -3072,7 +3175,8 @@ OrphanBuilder OrphanBuilder::initText(
OrphanBuilder OrphanBuilder::initData(
BuilderArena* arena, CapTableBuilder* capTable, ByteCount size) {
OrphanBuilder result;
auto allocation = WireHelpers::initDataPointer(result.tagAsPtr(), nullptr, capTable, size, arena);
auto allocation = WireHelpers::initDataPointer(result.tagAsPtr(), nullptr, capTable,
assertMaxBits<BLOB_SIZE_BITS>(size), arena);
result.segment = allocation.segment;
result.capTable = capTable;
result.location = reinterpret_cast<word*>(allocation.value.begin());
......@@ -3236,12 +3340,14 @@ OrphanBuilder OrphanBuilder::referenceExternalData(BuilderArena* arena, Data::Re
KJ_REQUIRE(reinterpret_cast<uintptr_t>(data.begin()) % sizeof(void*) == 0,
"Cannot referenceExternalData() that is not aligned.");
auto wordCount = WireHelpers::roundBytesUpToWords(data.size() * BYTES);
kj::ArrayPtr<const word> words(reinterpret_cast<const word*>(data.begin()), wordCount / WORDS);
auto checkedSize = assertMaxBits<BLOB_SIZE_BITS>(guarded(data.size()));
auto wordCount = WireHelpers::roundBytesUpToWords(checkedSize * BYTES);
kj::ArrayPtr<const word> words(reinterpret_cast<const word*>(data.begin()),
unguard(wordCount / WORDS));
OrphanBuilder result;
result.tagAsPtr()->setKindForOrphan(WirePointer::LIST);
result.tagAsPtr()->listRef.set(ElementSize::BYTE, data.size() * ELEMENTS);
result.tagAsPtr()->listRef.set(ElementSize::BYTE, checkedSize * ELEMENTS);
result.segment = arena->addExternalSegment(words);
// External data cannot possibly contain capabilities.
......@@ -3297,7 +3403,7 @@ Text::Builder OrphanBuilder::asText() {
// Never relocates.
return WireHelpers::getWritableTextPointer(
tagAsPtr(), location, segment, capTable, nullptr, 0 * BYTES);
tagAsPtr(), location, segment, capTable, nullptr, ZERO * BYTES);
}
Data::Builder OrphanBuilder::asData() {
......@@ -3305,7 +3411,7 @@ Data::Builder OrphanBuilder::asData() {
// Never relocates.
return WireHelpers::getWritableDataPointer(
tagAsPtr(), location, segment, capTable, nullptr, 0 * BYTES);
tagAsPtr(), location, segment, capTable, nullptr, ZERO * BYTES);
}
StructReader OrphanBuilder::asStructReader(StructSize size) const {
......@@ -3328,12 +3434,12 @@ kj::Own<ClientHook> OrphanBuilder::asCapability() const {
Text::Reader OrphanBuilder::asTextReader() const {
KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
return WireHelpers::readTextPointer(segment, tagAsPtr(), location, nullptr, 0 * BYTES);
return WireHelpers::readTextPointer(segment, tagAsPtr(), location, nullptr, ZERO * BYTES);
}
Data::Reader OrphanBuilder::asDataReader() const {
KJ_DASSERT(tagAsPtr()->isNull() == (location == nullptr));
return WireHelpers::readDataPointer(segment, tagAsPtr(), location, nullptr, 0 * BYTES);
return WireHelpers::readDataPointer(segment, tagAsPtr(), location, nullptr, ZERO * BYTES);
}
bool OrphanBuilder::truncate(ElementCount size, bool isText) {
......
c++/src/capnp/layout.h
View file @ e2cbc1b5
......@@ -82,26 +82,48 @@ class BuilderArena;
// =============================================================================
typedef decltype(BITS / ELEMENTS) BitsPerElement;
typedef decltype(POINTERS / ELEMENTS) PointersPerElement;
static constexpr BitsPerElement BITS_PER_ELEMENT_TABLE[8] = {
0 * BITS / ELEMENTS,
1 * BITS / ELEMENTS,
8 * BITS / ELEMENTS,
16 * BITS / ELEMENTS,
32 * BITS / ELEMENTS,
64 * BITS / ELEMENTS,
0 * BITS / ELEMENTS,
0 * BITS / ELEMENTS
#if CAPNP_DEBUG_TYPES
typedef kj::UnitRatio<kj::Guarded<64, uint>, BitLabel, ElementLabel> BitsPerElementTableType;
#else
typedef uint BitsPerElementTableType;
#endif
static constexpr BitsPerElementTableType BITS_PER_ELEMENT_TABLE[8] = {
guarded< 0>() * BITS / ELEMENTS,
guarded< 1>() * BITS / ELEMENTS,
guarded< 8>() * BITS / ELEMENTS,
guarded<16>() * BITS / ELEMENTS,
guarded<32>() * BITS / ELEMENTS,
guarded<64>() * BITS / ELEMENTS,
guarded< 0>() * BITS / ELEMENTS,
guarded< 0>() * BITS / ELEMENTS
};
inline KJ_CONSTEXPR() BitsPerElement dataBitsPerElement(ElementSize size) {
inline KJ_CONSTEXPR() BitsPerElementTableType dataBitsPerElement(ElementSize size) {
return _::BITS_PER_ELEMENT_TABLE[static_cast<int>(size)];
}
inline constexpr PointersPerElement pointersPerElement(ElementSize size) {
return size == ElementSize::POINTER ? 1 * POINTERS / ELEMENTS : 0 * POINTERS / ELEMENTS;
inline constexpr PointersPerElementN<1> pointersPerElement(ElementSize size) {
if (size == ElementSize::POINTER) {
return ONE * POINTERS / ELEMENTS;
} else {
return ZERO * POINTERS / ELEMENTS;
}
}
static constexpr BitsPerElementTableType BITS_PER_ELEMENT_INCLUDING_PONITERS_TABLE[8] = {
guarded< 0>() * BITS / ELEMENTS,
guarded< 1>() * BITS / ELEMENTS,
guarded< 8>() * BITS / ELEMENTS,
guarded<16>() * BITS / ELEMENTS,
guarded<32>() * BITS / ELEMENTS,
guarded<64>() * BITS / ELEMENTS,
guarded<64>() * BITS / ELEMENTS,
guarded< 0>() * BITS / ELEMENTS
};
inline KJ_CONSTEXPR() BitsPerElementTableType bitsPerElementIncludingPointers(ElementSize size) {
return _::BITS_PER_ELEMENT_INCLUDING_PONITERS_TABLE[static_cast<int>(size)];
}
template <size_t size> struct ElementSizeForByteSize;
......@@ -142,17 +164,23 @@ inline constexpr ElementSize elementSizeForType() {
}
struct MessageSizeCounts {
WordCount64 wordCount;
WordCountN<61, uint64_t> wordCount; // 2^64 bytes
uint capCount;
MessageSizeCounts& operator+=(const MessageSizeCounts& other) {
wordCount += other.wordCount;
// OK to truncate unchecked because this class is used to count actual stuff in memory, and
// we couldn't possibly have anywhere near 2^61 words.
wordCount = assumeBits<61>(wordCount + other.wordCount);
capCount += other.capCount;
return *this;
}
void addWords(WordCountN<61, uint64_t> other) {
wordCount = assumeBits<61>(wordCount + other);
}
MessageSize asPublic() {
return MessageSize { wordCount / WORDS, capCount };
return MessageSize { unguard(wordCount / WORDS), capCount };
}
};
......@@ -168,13 +196,13 @@ union AlignedData {
};
struct StructSize {
WordCount16 data;
WirePointerCount16 pointers;
StructDataWordCount data;
StructPointerCount pointers;
inline constexpr WordCount total() const { return data + pointers * WORDS_PER_POINTER; }
inline constexpr WordCountN<17> total() const { return data + pointers * WORDS_PER_POINTER; }
StructSize() = default;
inline constexpr StructSize(WordCount data, WirePointerCount pointers)
inline constexpr StructSize(StructDataWordCount data, StructPointerCount pointers)
: data(data), pointers(pointers) {}
};
......@@ -324,7 +352,8 @@ public:
ListBuilder getList(ElementSize elementSize, const word* defaultValue);
ListBuilder getStructList(StructSize elementSize, const word* defaultValue);
ListBuilder getListAnySize(const word* defaultValue);
template <typename T> typename T::Builder getBlob(const void* defaultValue,ByteCount defaultSize);
template <typename T> typename T::Builder getBlob(
const void* defaultValue, ByteCount defaultSize);
#if !CAPNP_LITE
kj::Own<ClientHook> getCapability();
#endif // !CAPNP_LITE
......@@ -474,37 +503,36 @@ public:
// Get the object's location. Only valid for independently-allocated objects (i.e. not list
// elements).
inline BitCount getDataSectionSize() const { return dataSize; }
inline WirePointerCount getPointerSectionSize() const { return pointerCount; }
inline StructDataBitCount getDataSectionSize() const { return dataSize; }
inline StructPointerCount getPointerSectionSize() const { return pointerCount; }
inline kj::ArrayPtr<byte> getDataSectionAsBlob();
inline _::ListBuilder getPointerSectionAsList();
template <typename T>
KJ_ALWAYS_INLINE(bool hasDataField(ElementCount offset));
KJ_ALWAYS_INLINE(bool hasDataField(StructDataElementOffset offset));
// Return true if the field is set to something other than its default value.
template <typename T>
KJ_ALWAYS_INLINE(T getDataField(ElementCount offset));
KJ_ALWAYS_INLINE(T getDataField(StructDataElementOffset offset));
// Gets the data field value of the given type at the given offset. The offset is measured in
// multiples of the field size, determined by the type.
template <typename T>
KJ_ALWAYS_INLINE(T getDataField(ElementCount offset, Mask<T> mask));
KJ_ALWAYS_INLINE(T getDataField(StructDataElementOffset offset, Mask<T> mask));
// Like getDataField() but applies the given XOR mask to the data on load. Used for reading
// fields with non-zero default values.
template <typename T>
KJ_ALWAYS_INLINE(void setDataField(
ElementCount offset, kj::NoInfer<T> value));
KJ_ALWAYS_INLINE(void setDataField(StructDataElementOffset offset, kj::NoInfer<T> value));
// Sets the data field value at the given offset.
template <typename T>
KJ_ALWAYS_INLINE(void setDataField(
ElementCount offset, kj::NoInfer<T> value, Mask<T> mask));
KJ_ALWAYS_INLINE(void setDataField(StructDataElementOffset offset,
kj::NoInfer<T> value, Mask<T> mask));
// Like setDataField() but applies the given XOR mask before storing. Used for writing fields
// with non-zero default values.
KJ_ALWAYS_INLINE(PointerBuilder getPointerField(WirePointerCount ptrIndex));
KJ_ALWAYS_INLINE(PointerBuilder getPointerField(StructPointerCount ptrIndex));
// Get a builder for a pointer field given the index within the pointer section.
void clearAll();
......@@ -538,15 +566,15 @@ private:
void* data; // Pointer to the encoded data.
WirePointer* pointers; // Pointer to the encoded pointers.
BitCount32 dataSize;
StructDataBitCount dataSize;
// Size of data section. We use a bit count rather than a word count to more easily handle the
// case of struct lists encoded with less than a word per element.
WirePointerCount16 pointerCount; // Size of the pointer section.
StructPointerCount pointerCount; // Size of the pointer section.
inline StructBuilder(SegmentBuilder* segment, CapTableBuilder* capTable,
void* data, WirePointer* pointers,
BitCount dataSize, WirePointerCount pointerCount)
StructDataBitCount dataSize, StructPointerCount pointerCount)
: segment(segment), capTable(capTable), data(data), pointers(pointers),
dataSize(dataSize), pointerCount(pointerCount) {}
......@@ -558,38 +586,38 @@ private:
class StructReader {
public:
inline StructReader()
: segment(nullptr), capTable(nullptr), data(nullptr), pointers(nullptr), dataSize(0),
pointerCount(0), nestingLimit(0x7fffffff) {}
: segment(nullptr), capTable(nullptr), data(nullptr), pointers(nullptr),
dataSize(ZERO * BITS), pointerCount(ZERO * POINTERS), nestingLimit(0x7fffffff) {}
inline StructReader(kj::ArrayPtr<const word> data)
: segment(nullptr), capTable(nullptr), data(data.begin()), pointers(nullptr),
dataSize(data.size() * WORDS * BITS_PER_WORD), pointerCount(0), nestingLimit(0x7fffffff) {}
dataSize(data.size() * WORDS * BITS_PER_WORD), pointerCount(ZERO * POINTERS),
nestingLimit(0x7fffffff) {}
const void* getLocation() const { return data; }
inline BitCount getDataSectionSize() const { return dataSize; }
inline WirePointerCount getPointerSectionSize() const { return pointerCount; }
inline StructDataBitCount getDataSectionSize() const { return dataSize; }
inline StructPointerCount getPointerSectionSize() const { return pointerCount; }
inline kj::ArrayPtr<const byte> getDataSectionAsBlob();
inline _::ListReader getPointerSectionAsList();
kj::Array<word> canonicalize();
template <typename T>
KJ_ALWAYS_INLINE(bool hasDataField(ElementCount offset) const);
KJ_ALWAYS_INLINE(bool hasDataField(StructDataElementOffset offset) const);
// Return true if the field is set to something other than its default value.
template <typename T>
KJ_ALWAYS_INLINE(T getDataField(ElementCount offset) const);
KJ_ALWAYS_INLINE(T getDataField(StructDataElementOffset offset) const);
// Get the data field value of the given type at the given offset. The offset is measured in
// multiples of the field size, determined by the type. Returns zero if the offset is past the
// end of the struct's data section.
template <typename T>
KJ_ALWAYS_INLINE(
T getDataField(ElementCount offset, Mask<T> mask) const);
KJ_ALWAYS_INLINE(T getDataField(StructDataElementOffset offset, Mask<T> mask) const);
// Like getDataField(offset), but applies the given XOR mask to the result. Used for reading
// fields with non-zero default values.
KJ_ALWAYS_INLINE(PointerReader getPointerField(WirePointerCount ptrIndex) const);
KJ_ALWAYS_INLINE(PointerReader getPointerField(StructPointerCount ptrIndex) const);
// Get a reader for a pointer field given the index within the pointer section. If the index
// is out-of-bounds, returns a null pointer.
......@@ -628,11 +656,11 @@ private:
const void* data;
const WirePointer* pointers;
BitCount32 dataSize;
StructDataBitCount dataSize;
// Size of data section. We use a bit count rather than a word count to more easily handle the
// case of struct lists encoded with less than a word per element.
WirePointerCount16 pointerCount; // Size of the pointer section.
StructPointerCount pointerCount; // Size of the pointer section.
int nestingLimit;
// Limits the depth of message structures to guard against stack-overflow-based DoS attacks.
......@@ -641,7 +669,8 @@ private:
inline StructReader(SegmentReader* segment, CapTableReader* capTable,
const void* data, const WirePointer* pointers,
BitCount dataSize, WirePointerCount pointerCount, int nestingLimit)
StructDataBitCount dataSize, StructPointerCount pointerCount,
int nestingLimit)
: segment(segment), capTable(capTable), data(data), pointers(pointers),
dataSize(dataSize), pointerCount(pointerCount),
nestingLimit(nestingLimit) {}
......@@ -657,8 +686,8 @@ class ListBuilder: public kj::DisallowConstCopy {
public:
inline explicit ListBuilder(ElementSize elementSize)
: segment(nullptr), capTable(nullptr), ptr(nullptr), elementCount(0 * ELEMENTS),
step(0 * BITS / ELEMENTS), structDataSize(0 * BITS), structPointerCount(0 * POINTERS),
elementSize(elementSize) {}
step(ZERO * BITS / ELEMENTS), structDataSize(0 * BITS),
structPointerCount(ZERO * POINTERS), elementSize(elementSize) {}
inline word* getLocation() {
// Get the object's location.
......@@ -672,7 +701,7 @@ public:
inline ElementSize getElementSize() const { return elementSize; }
inline ElementCount size() const;
inline ListElementCount size() const;
// The number of elements in the list.
Text::Builder asText();
......@@ -684,8 +713,7 @@ public:
// Get the element of the given type at the given index.
template <typename T>
KJ_ALWAYS_INLINE(void setDataElement(
ElementCount index, kj::NoInfer<T> value));
KJ_ALWAYS_INLINE(void setDataElement(ElementCount index, kj::NoInfer<T> value));
// Set the element at the given index.
KJ_ALWAYS_INLINE(PointerBuilder getPointerElement(ElementCount index));
......@@ -710,13 +738,14 @@ private:
byte* ptr; // Pointer to list content.
ElementCount elementCount; // Number of elements in the list.
ListElementCount elementCount; // Number of elements in the list.
decltype(BITS / ELEMENTS) step;
// The distance between elements.
BitsPerElementN<23> step;
// The distance between elements. The maximum value occurs when a struct contains 2^16-1 data
// words and 2^16-1 pointers, i.e. 2^17 - 2 words, or 2^23 - 128 bits.
BitCount32 structDataSize;
WirePointerCount16 structPointerCount;
StructDataBitCount structDataSize;
StructPointerCount structPointerCount;
// The struct properties to use when interpreting the elements as structs. All lists can be
// interpreted as struct lists, so these are always filled in.
......@@ -726,7 +755,7 @@ private:
inline ListBuilder(SegmentBuilder* segment, CapTableBuilder* capTable, void* ptr,
decltype(BITS / ELEMENTS) step, ElementCount size,
BitCount structDataSize, WirePointerCount structPointerCount,
StructDataBitCount structDataSize, StructPointerCount structPointerCount,
ElementSize elementSize)
: segment(segment), capTable(capTable), ptr(reinterpret_cast<byte*>(ptr)),
elementCount(size), step(step), structDataSize(structDataSize),
......@@ -740,11 +769,11 @@ private:
class ListReader {
public:
inline explicit ListReader(ElementSize elementSize)
: segment(nullptr), capTable(nullptr), ptr(nullptr), elementCount(0),
step(0 * BITS / ELEMENTS), structDataSize(0), structPointerCount(0),
elementSize(elementSize), nestingLimit(0x7fffffff) {}
: segment(nullptr), capTable(nullptr), ptr(nullptr), elementCount(ZERO * ELEMENTS),
step(ZERO * BITS / ELEMENTS), structDataSize(ZERO * BITS),
structPointerCount(ZERO * POINTERS), elementSize(elementSize), nestingLimit(0x7fffffff) {}
inline ElementCount size() const;
inline ListElementCount size() const;
// The number of elements in the list.
inline ElementSize getElementSize() const { return elementSize; }
......@@ -782,13 +811,14 @@ private:
const byte* ptr; // Pointer to list content.
ElementCount elementCount; // Number of elements in the list.
ListElementCount elementCount; // Number of elements in the list.
decltype(BITS / ELEMENTS) step;
// The distance between elements.
BitsPerElementN<23> step;
// The distance between elements. The maximum value occurs when a struct contains 2^16-1 data
// words and 2^16-1 pointers, i.e. 2^17 - 2 words, or 2^23 - 2 bits.
BitCount32 structDataSize;
WirePointerCount16 structPointerCount;
StructDataBitCount structDataSize;
StructPointerCount structPointerCount;
// The struct properties to use when interpreting the elements as structs. All lists can be
// interpreted as struct lists, so these are always filled in.
......@@ -801,8 +831,8 @@ private:
// Once this reaches zero, further pointers will be pruned.
inline ListReader(SegmentReader* segment, CapTableReader* capTable, const void* ptr,
ElementCount elementCount, decltype(BITS / ELEMENTS) step,
BitCount structDataSize, WirePointerCount structPointerCount,
ElementCount elementCount, BitsPerElementN<23> step,
StructDataBitCount structDataSize, StructPointerCount structPointerCount,
ElementSize elementSize, int nestingLimit)
: segment(segment), capTable(capTable), ptr(reinterpret_cast<const byte*>(ptr)),
elementCount(elementCount), step(step), structDataSize(structDataSize),
......@@ -888,7 +918,7 @@ public:
// Versions of truncate() that know how to allocate a new list if needed.
private:
static_assert(1 * POINTERS * WORDS_PER_POINTER == 1 * WORDS,
static_assert(ONE * POINTERS * WORDS_PER_POINTER == ONE * WORDS,
"This struct assumes a pointer is one word.");
word tag;
// Contains an encoded WirePointer representing this object. WirePointer is defined in
......@@ -934,13 +964,17 @@ private:
// These are defined in the source file.
template <> typename Text::Builder PointerBuilder::initBlob<Text>(ByteCount size);
template <> void PointerBuilder::setBlob<Text>(typename Text::Reader value);
template <> typename Text::Builder PointerBuilder::getBlob<Text>(const void* defaultValue, ByteCount defaultSize);
template <> typename Text::Reader PointerReader::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) const;
template <> typename Text::Builder PointerBuilder::getBlob<Text>(
const void* defaultValue, ByteCount defaultSize);
template <> typename Text::Reader PointerReader::getBlob<Text>(
const void* defaultValue, ByteCount defaultSize) const;
template <> typename Data::Builder PointerBuilder::initBlob<Data>(ByteCount size);
template <> void PointerBuilder::setBlob<Data>(typename Data::Reader value);
template <> typename Data::Builder PointerBuilder::getBlob<Data>(const void* defaultValue, ByteCount defaultSize);
template <> typename Data::Reader PointerReader::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) const;
template <> typename Data::Builder PointerBuilder::getBlob<Data>(
const void* defaultValue, ByteCount defaultSize);
template <> typename Data::Reader PointerReader::getBlob<Data>(
const void* defaultValue, ByteCount defaultSize) const;
inline PointerBuilder PointerBuilder::getRoot(
SegmentBuilder* segment, CapTableBuilder* capTable, word* location) {
......@@ -955,82 +989,85 @@ inline PointerReader PointerReader::getRootUnchecked(const word* location) {
// -------------------------------------------------------------------
inline kj::ArrayPtr<byte> StructBuilder::getDataSectionAsBlob() {
return kj::ArrayPtr<byte>(reinterpret_cast<byte*>(data), dataSize / BITS_PER_BYTE / BYTES);
return kj::ArrayPtr<byte>(reinterpret_cast<byte*>(data),
unguard(dataSize / BITS_PER_BYTE / BYTES));
}
inline _::ListBuilder StructBuilder::getPointerSectionAsList() {
return _::ListBuilder(segment, capTable, pointers, 1 * POINTERS * BITS_PER_POINTER / ELEMENTS,
pointerCount * (1 * ELEMENTS / POINTERS),
0 * BITS, 1 * POINTERS, ElementSize::POINTER);
return _::ListBuilder(segment, capTable, pointers, ONE * POINTERS * BITS_PER_POINTER / ELEMENTS,
pointerCount * (ONE * ELEMENTS / POINTERS),
ZERO * BITS, ONE * POINTERS, ElementSize::POINTER);
}
template <typename T>
inline bool StructBuilder::hasDataField(ElementCount offset) {
inline bool StructBuilder::hasDataField(StructDataElementOffset offset) {
return getDataField<Mask<T>>(offset) != 0;
}
template <>
inline bool StructBuilder::hasDataField<Void>(ElementCount offset) {
inline bool StructBuilder::hasDataField<Void>(StructDataElementOffset offset) {
return false;
}
template <typename T>
inline T StructBuilder::getDataField(ElementCount offset) {
inline T StructBuilder::getDataField(StructDataElementOffset offset) {
return reinterpret_cast<WireValue<T>*>(data)[offset / ELEMENTS].get();
}
template <>
inline bool StructBuilder::getDataField<bool>(ElementCount offset) {
BitCount boffset = offset * (1 * BITS / ELEMENTS);
inline bool StructBuilder::getDataField<bool>(StructDataElementOffset offset) {
BitCountN<22> boffset = offset * (ONE * BITS / ELEMENTS);
byte* b = reinterpret_cast<byte*>(data) + boffset / BITS_PER_BYTE;
return (*reinterpret_cast<uint8_t*>(b) & (1 << (boffset % BITS_PER_BYTE / BITS))) != 0;
return (*reinterpret_cast<uint8_t*>(b) &
unguard(ONE << (boffset % BITS_PER_BYTE / BITS))) != 0;
}
template <>
inline Void StructBuilder::getDataField<Void>(ElementCount offset) {
inline Void StructBuilder::getDataField<Void>(StructDataElementOffset offset) {
return VOID;
}
template <typename T>
inline T StructBuilder::getDataField(ElementCount offset, Mask<T> mask) {
inline T StructBuilder::getDataField(StructDataElementOffset offset, Mask<T> mask) {
return unmask<T>(getDataField<Mask<T> >(offset), mask);
}
template <typename T>
inline void StructBuilder::setDataField(ElementCount offset, kj::NoInfer<T> value) {
inline void StructBuilder::setDataField(StructDataElementOffset offset, kj::NoInfer<T> value) {
reinterpret_cast<WireValue<T>*>(data)[offset / ELEMENTS].set(value);
}
#if CAPNP_CANONICALIZE_NAN
// Use mask() on floats and doubles to make sure we canonicalize NaNs.
template <>
inline void StructBuilder::setDataField<float>(ElementCount offset, float value) {
inline void StructBuilder::setDataField<float>(StructDataElementOffset offset, float value) {
setDataField<uint32_t>(offset, mask<float>(value, 0));
}
template <>
inline void StructBuilder::setDataField<double>(ElementCount offset, double value) {
inline void StructBuilder::setDataField<double>(StructDataElementOffset offset, double value) {
setDataField<uint64_t>(offset, mask<double>(value, 0));
}
#endif
template <>
inline void StructBuilder::setDataField<bool>(ElementCount offset, bool value) {
BitCount boffset = offset * (1 * BITS / ELEMENTS);
inline void StructBuilder::setDataField<bool>(StructDataElementOffset offset, bool value) {
auto boffset = offset * (ONE * BITS / ELEMENTS);
byte* b = reinterpret_cast<byte*>(data) + boffset / BITS_PER_BYTE;
uint bitnum = boffset % BITS_PER_BYTE / BITS;
uint bitnum = unguardMaxBits<3>(boffset % BITS_PER_BYTE / BITS);
*reinterpret_cast<uint8_t*>(b) = (*reinterpret_cast<uint8_t*>(b) & ~(1 << bitnum))
| (static_cast<uint8_t>(value) << bitnum);
}
template <>
inline void StructBuilder::setDataField<Void>(ElementCount offset, Void value) {}
inline void StructBuilder::setDataField<Void>(StructDataElementOffset offset, Void value) {}
template <typename T>
inline void StructBuilder::setDataField(ElementCount offset, kj::NoInfer<T> value, Mask<T> m) {
inline void StructBuilder::setDataField(StructDataElementOffset offset,
kj::NoInfer<T> value, Mask<T> m) {
setDataField<Mask<T> >(offset, mask<T>(value, m));
}
inline PointerBuilder StructBuilder::getPointerField(WirePointerCount ptrIndex) {
inline PointerBuilder StructBuilder::getPointerField(StructPointerCount ptrIndex) {
// Hacky because WirePointer is defined in the .c++ file (so is incomplete here).
return PointerBuilder(segment, capTable, reinterpret_cast<WirePointer*>(
reinterpret_cast<word*>(pointers) + ptrIndex * WORDS_PER_POINTER));
......@@ -1039,28 +1076,29 @@ inline PointerBuilder StructBuilder::getPointerField(WirePointerCount ptrIndex)
// -------------------------------------------------------------------
inline kj::ArrayPtr<const byte> StructReader::getDataSectionAsBlob() {
return kj::ArrayPtr<const byte>(reinterpret_cast<const byte*>(data), dataSize / BITS_PER_BYTE / BYTES);
return kj::ArrayPtr<const byte>(reinterpret_cast<const byte*>(data),
unguard(dataSize / BITS_PER_BYTE / BYTES));
}
inline _::ListReader StructReader::getPointerSectionAsList() {
return _::ListReader(segment, capTable, pointers, pointerCount * (1 * ELEMENTS / POINTERS),
1 * POINTERS * BITS_PER_POINTER / ELEMENTS, 0 * BITS, 1 * POINTERS,
return _::ListReader(segment, capTable, pointers, pointerCount * (ONE * ELEMENTS / POINTERS),
ONE * POINTERS * BITS_PER_POINTER / ELEMENTS, ZERO * BITS, ONE * POINTERS,
ElementSize::POINTER, nestingLimit);
}
template <typename T>
inline bool StructReader::hasDataField(ElementCount offset) const {
inline bool StructReader::hasDataField(StructDataElementOffset offset) const {
return getDataField<Mask<T>>(offset) != 0;
}
template <>
inline bool StructReader::hasDataField<Void>(ElementCount offset) const {
inline bool StructReader::hasDataField<Void>(StructDataElementOffset offset) const {
return false;
}
template <typename T>
inline T StructReader::getDataField(ElementCount offset) const {
if ((offset + 1 * ELEMENTS) * capnp::bitsPerElement<T>() <= dataSize) {
inline T StructReader::getDataField(StructDataElementOffset offset) const {
if ((offset + ONE * ELEMENTS) * capnp::bitsPerElement<T>() <= dataSize) {
return reinterpret_cast<const WireValue<T>*>(data)[offset / ELEMENTS].get();
} else {
return static_cast<T>(0);
......@@ -1068,27 +1106,28 @@ inline T StructReader::getDataField(ElementCount offset) const {
}
template <>
inline bool StructReader::getDataField<bool>(ElementCount offset) const {
BitCount boffset = offset * (1 * BITS / ELEMENTS);
inline bool StructReader::getDataField<bool>(StructDataElementOffset offset) const {
auto boffset = offset * (ONE * BITS / ELEMENTS);
if (boffset < dataSize) {
const byte* b = reinterpret_cast<const byte*>(data) + boffset / BITS_PER_BYTE;
return (*reinterpret_cast<const uint8_t*>(b) & (1 << (boffset % BITS_PER_BYTE / BITS))) != 0;
return (*reinterpret_cast<const uint8_t*>(b) &
unguard(ONE << (boffset % BITS_PER_BYTE / BITS))) != 0;
} else {
return false;
}
}
template <>
inline Void StructReader::getDataField<Void>(ElementCount offset) const {
inline Void StructReader::getDataField<Void>(StructDataElementOffset offset) const {
return VOID;
}
template <typename T>
T StructReader::getDataField(ElementCount offset, Mask<T> mask) const {
T StructReader::getDataField(StructDataElementOffset offset, Mask<T> mask) const {
return unmask<T>(getDataField<Mask<T> >(offset), mask);
}
inline PointerReader StructReader::getPointerField(WirePointerCount ptrIndex) const {
inline PointerReader StructReader::getPointerField(StructPointerCount ptrIndex) const {
if (ptrIndex < pointerCount) {
// Hacky because WirePointer is defined in the .c++ file (so is incomplete here).
return PointerReader(segment, capTable, reinterpret_cast<const WirePointer*>(
......@@ -1100,11 +1139,12 @@ inline PointerReader StructReader::getPointerField(WirePointerCount ptrIndex) co
// -------------------------------------------------------------------
inline ElementCount ListBuilder::size() const { return elementCount; }
inline ListElementCount ListBuilder::size() const { return elementCount; }
template <typename T>
inline T ListBuilder::getDataElement(ElementCount index) {
return reinterpret_cast<WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->get();
return reinterpret_cast<WireValue<T>*>(
ptr + upgradeGuard<uint64_t>(index) * step / BITS_PER_BYTE)->get();
// TODO(perf): Benchmark this alternate implementation, which I suspect may make better use of
// the x86 SIB byte. Also use it for all the other getData/setData implementations below, and
......@@ -1117,9 +1157,10 @@ inline T ListBuilder::getDataElement(ElementCount index) {
template <>
inline bool ListBuilder::getDataElement<bool>(ElementCount index) {
// Ignore step for bit lists because bit lists cannot be upgraded to struct lists.
BitCount bindex = index * (1 * BITS / ELEMENTS);
auto bindex = index * (ONE * BITS / ELEMENTS);
byte* b = ptr + bindex / BITS_PER_BYTE;
return (*reinterpret_cast<uint8_t*>(b) & (1 << (bindex % BITS_PER_BYTE / BITS))) != 0;
return (*reinterpret_cast<uint8_t*>(b) &
unguard(ONE << (bindex % BITS_PER_BYTE / BITS))) != 0;
}
template <>
......@@ -1129,7 +1170,8 @@ inline Void ListBuilder::getDataElement<Void>(ElementCount index) {
template <typename T>
inline void ListBuilder::setDataElement(ElementCount index, kj::NoInfer<T> value) {
reinterpret_cast<WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->set(value);
reinterpret_cast<WireValue<T>*>(
ptr + upgradeGuard<uint64_t>(index) * step / BITS_PER_BYTE)->set(value);
}
#if CAPNP_CANONICALIZE_NAN
......@@ -1147,36 +1189,38 @@ inline void ListBuilder::setDataElement<double>(ElementCount index, double value
template <>
inline void ListBuilder::setDataElement<bool>(ElementCount index, bool value) {
// Ignore stepBytes for bit lists because bit lists cannot be upgraded to struct lists.
BitCount bindex = index * (1 * BITS / ELEMENTS);
auto bindex = index * (ONE * BITS / ELEMENTS);
byte* b = ptr + bindex / BITS_PER_BYTE;
uint bitnum = bindex % BITS_PER_BYTE / BITS;
*reinterpret_cast<uint8_t*>(b) = (*reinterpret_cast<uint8_t*>(b) & ~(1 << bitnum))
| (static_cast<uint8_t>(value) << bitnum);
auto bitnum = bindex % BITS_PER_BYTE / BITS;
*reinterpret_cast<uint8_t*>(b) = (*reinterpret_cast<uint8_t*>(b) & ~(1 << unguard(bitnum)))
| (static_cast<uint8_t>(value) << unguard(bitnum));
}
template <>
inline void ListBuilder::setDataElement<Void>(ElementCount index, Void value) {}
inline PointerBuilder ListBuilder::getPointerElement(ElementCount index) {
return PointerBuilder(segment, capTable,
reinterpret_cast<WirePointer*>(ptr + index * step / BITS_PER_BYTE));
return PointerBuilder(segment, capTable, reinterpret_cast<WirePointer*>(ptr +
upgradeGuard<uint64_t>(index) * step / BITS_PER_BYTE));
}
// -------------------------------------------------------------------
inline ElementCount ListReader::size() const { return elementCount; }
inline ListElementCount ListReader::size() const { return elementCount; }
template <typename T>
inline T ListReader::getDataElement(ElementCount index) const {
return reinterpret_cast<const WireValue<T>*>(ptr + index * step / BITS_PER_BYTE)->get();
return reinterpret_cast<const WireValue<T>*>(
ptr + upgradeGuard<uint64_t>(index) * step / BITS_PER_BYTE)->get();
}
template <>
inline bool ListReader::getDataElement<bool>(ElementCount index) const {
// Ignore step for bit lists because bit lists cannot be upgraded to struct lists.
BitCount bindex = index * (1 * BITS / ELEMENTS);
auto bindex = index * (ONE * BITS / ELEMENTS);
const byte* b = ptr + bindex / BITS_PER_BYTE;
return (*reinterpret_cast<const uint8_t*>(b) & (1 << (bindex % BITS_PER_BYTE / BITS))) != 0;
return (*reinterpret_cast<const uint8_t*>(b) &
unguard(ONE << (bindex % BITS_PER_BYTE / BITS))) != 0;
}
template <>
......@@ -1185,8 +1229,8 @@ inline Void ListReader::getDataElement<Void>(ElementCount index) const {
}
inline PointerReader ListReader::getPointerElement(ElementCount index) const {
return PointerReader(segment, capTable,
reinterpret_cast<const WirePointer*>(ptr + index * step / BITS_PER_BYTE), nestingLimit);
return PointerReader(segment, capTable, reinterpret_cast<const WirePointer*>(
ptr + upgradeGuard<uint64_t>(index) * step / BITS_PER_BYTE), nestingLimit);
}
// -------------------------------------------------------------------
......
c++/src/capnp/list.h
View file @ e2cbc1b5
......@@ -117,7 +117,7 @@ struct List<T, Kind::PRIMITIVE> {
inline uint size() const { return reader.size() / ELEMENTS; }
inline T operator[](uint index) const {
KJ_IREQUIRE(index < size());
return reader.template getDataElement<T>(index * ELEMENTS);
return reader.template getDataElement<T>(guarded(index) * ELEMENTS);
}
typedef _::IndexingIterator<const Reader, T> Iterator;
......@@ -149,7 +149,7 @@ struct List<T, Kind::PRIMITIVE> {
inline uint size() const { return builder.size() / ELEMENTS; }
inline T operator[](uint index) {
KJ_IREQUIRE(index < size());
return builder.template getDataElement<T>(index * ELEMENTS);
return builder.template getDataElement<T>(guarded(index) * ELEMENTS);
}
inline void set(uint index, T value) {
// Alas, it is not possible to make operator[] return a reference to which you can assign,
......@@ -158,7 +158,7 @@ struct List<T, Kind::PRIMITIVE> {
// operator=() because it will lead to surprising behavior when using type inference (e.g.
// calling a template function with inferred argument types, or using "auto" or "decltype").
builder.template setDataElement<T>(index * ELEMENTS, value);
builder.template setDataElement<T>(guarded(index) * ELEMENTS, value);
}
typedef _::IndexingIterator<Builder, T> Iterator;
......@@ -178,7 +178,7 @@ struct List<T, Kind::PRIMITIVE> {
private:
inline static _::ListBuilder initPointer(_::PointerBuilder builder, uint size) {
return builder.initList(_::elementSizeForType<T>(), size * ELEMENTS);
return builder.initList(_::elementSizeForType<T>(), guarded(size) * ELEMENTS);
}
inline static _::ListBuilder getFromPointer(_::PointerBuilder builder, const word* defaultValue) {
return builder.getList(_::elementSizeForType<T>(), defaultValue);
......@@ -213,7 +213,7 @@ struct List<T, Kind::STRUCT> {
inline uint size() const { return reader.size() / ELEMENTS; }
inline typename T::Reader operator[](uint index) const {
KJ_IREQUIRE(index < size());
return typename T::Reader(reader.getStructElement(index * ELEMENTS));
return typename T::Reader(reader.getStructElement(guarded(index) * ELEMENTS));
}
typedef _::IndexingIterator<const Reader, typename T::Reader> Iterator;
......@@ -245,7 +245,7 @@ struct List<T, Kind::STRUCT> {
inline uint size() const { return builder.size() / ELEMENTS; }
inline typename T::Builder operator[](uint index) {
KJ_IREQUIRE(index < size());
return typename T::Builder(builder.getStructElement(index * ELEMENTS));
return typename T::Builder(builder.getStructElement(guarded(index) * ELEMENTS));
}
inline void adoptWithCaveats(uint index, Orphan<T>&& orphan) {
......@@ -263,8 +263,8 @@ struct List<T, Kind::STRUCT> {
// We pass a zero-valued StructSize to asStruct() because we do not want the struct to be
// expanded under any circumstances. We're just going to throw it away anyway, and
// transferContentFrom() already carefully compares the struct sizes before transferring.
builder.getStructElement(index * ELEMENTS).transferContentFrom(
orphan.builder.asStruct(_::StructSize(0 * WORDS, 0 * POINTERS)));
builder.getStructElement(guarded(index) * ELEMENTS).transferContentFrom(
orphan.builder.asStruct(_::StructSize(ZERO * WORDS, ZERO * POINTERS)));
}
inline void setWithCaveats(uint index, const typename T::Reader& reader) {
// Mostly behaves like you'd expect `set` to behave, but with a caveat originating from
......@@ -278,7 +278,7 @@ struct List<T, Kind::STRUCT> {
// protocol. (Plus, it's easier to use anyhow.)
KJ_IREQUIRE(index < size());
builder.getStructElement(index * ELEMENTS).copyContentFrom(reader._reader);
builder.getStructElement(guarded(index) * ELEMENTS).copyContentFrom(reader._reader);
}
// There are no init(), set(), adopt(), or disown() methods for lists of structs because the
......@@ -303,7 +303,7 @@ struct List<T, Kind::STRUCT> {
private:
inline static _::ListBuilder initPointer(_::PointerBuilder builder, uint size) {
return builder.initStructList(size * ELEMENTS, _::structSize<T>());
return builder.initStructList(guarded(size) * ELEMENTS, _::structSize<T>());
}
inline static _::ListBuilder getFromPointer(_::PointerBuilder builder, const word* defaultValue) {
return builder.getStructList(_::structSize<T>(), defaultValue);
......@@ -335,8 +335,8 @@ struct List<List<T>, Kind::LIST> {
inline uint size() const { return reader.size() / ELEMENTS; }
inline typename List<T>::Reader operator[](uint index) const {
KJ_IREQUIRE(index < size());
return typename List<T>::Reader(
_::PointerHelpers<List<T>>::get(reader.getPointerElement(index * ELEMENTS)));
return typename List<T>::Reader(_::PointerHelpers<List<T>>::get(
reader.getPointerElement(guarded(index) * ELEMENTS)));
}
typedef _::IndexingIterator<const Reader, typename List<T>::Reader> Iterator;
......@@ -368,17 +368,17 @@ struct List<List<T>, Kind::LIST> {
inline uint size() const { return builder.size() / ELEMENTS; }
inline typename List<T>::Builder operator[](uint index) {
KJ_IREQUIRE(index < size());
return typename List<T>::Builder(
_::PointerHelpers<List<T>>::get(builder.getPointerElement(index * ELEMENTS)));
return typename List<T>::Builder(_::PointerHelpers<List<T>>::get(
builder.getPointerElement(guarded(index) * ELEMENTS)));
}
inline typename List<T>::Builder init(uint index, uint size) {
KJ_IREQUIRE(index < this->size());
return typename List<T>::Builder(
_::PointerHelpers<List<T>>::init(builder.getPointerElement(index * ELEMENTS), size));
return typename List<T>::Builder(_::PointerHelpers<List<T>>::init(
builder.getPointerElement(guarded(index) * ELEMENTS), size));
}
inline void set(uint index, typename List<T>::Reader value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).setList(value.reader);
builder.getPointerElement(guarded(index) * ELEMENTS).setList(value.reader);
}
void set(uint index, std::initializer_list<ReaderFor<T>> value) {
KJ_IREQUIRE(index < size());
......@@ -390,11 +390,11 @@ struct List<List<T>, Kind::LIST> {
}
inline void adopt(uint index, Orphan<T>&& value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value.builder));
builder.getPointerElement(guarded(index) * ELEMENTS).adopt(kj::mv(value.builder));
}
inline Orphan<T> disown(uint index) {
KJ_IREQUIRE(index < size());
return Orphan<T>(builder.getPointerElement(index * ELEMENTS).disown());
return Orphan<T>(builder.getPointerElement(guarded(index) * ELEMENTS).disown());
}
typedef _::IndexingIterator<Builder, typename List<T>::Builder> Iterator;
......@@ -414,7 +414,7 @@ struct List<List<T>, Kind::LIST> {
private:
inline static _::ListBuilder initPointer(_::PointerBuilder builder, uint size) {
return builder.initList(ElementSize::POINTER, size * ELEMENTS);
return builder.initList(ElementSize::POINTER, guarded(size) * ELEMENTS);
}
inline static _::ListBuilder getFromPointer(_::PointerBuilder builder, const word* defaultValue) {
return builder.getList(ElementSize::POINTER, defaultValue);
......@@ -444,7 +444,8 @@ struct List<T, Kind::BLOB> {
inline uint size() const { return reader.size() / ELEMENTS; }
inline typename T::Reader operator[](uint index) const {
KJ_IREQUIRE(index < size());
return reader.getPointerElement(index * ELEMENTS).template getBlob<T>(nullptr, 0 * BYTES);
return reader.getPointerElement(guarded(index) * ELEMENTS)
.template getBlob<T>(nullptr, ZERO * BYTES);
}
typedef _::IndexingIterator<const Reader, typename T::Reader> Iterator;
......@@ -476,23 +477,25 @@ struct List<T, Kind::BLOB> {
inline uint size() const { return builder.size() / ELEMENTS; }
inline typename T::Builder operator[](uint index) {
KJ_IREQUIRE(index < size());
return builder.getPointerElement(index * ELEMENTS).template getBlob<T>(nullptr, 0 * BYTES);
return builder.getPointerElement(guarded(index) * ELEMENTS)
.template getBlob<T>(nullptr, ZERO * BYTES);
}
inline void set(uint index, typename T::Reader value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).template setBlob<T>(value);
builder.getPointerElement(guarded(index) * ELEMENTS).template setBlob<T>(value);
}
inline typename T::Builder init(uint index, uint size) {
KJ_IREQUIRE(index < this->size());
return builder.getPointerElement(index * ELEMENTS).template initBlob<T>(size * BYTES);
return builder.getPointerElement(guarded(index) * ELEMENTS)
.template initBlob<T>(guarded(size) * BYTES);
}
inline void adopt(uint index, Orphan<T>&& value) {
KJ_IREQUIRE(index < size());
builder.getPointerElement(index * ELEMENTS).adopt(kj::mv(value.builder));
builder.getPointerElement(guarded(index) * ELEMENTS).adopt(kj::mv(value.builder));
}
inline Orphan<T> disown(uint index) {
KJ_IREQUIRE(index < size());
return Orphan<T>(builder.getPointerElement(index * ELEMENTS).disown());
return Orphan<T>(builder.getPointerElement(guarded(index) * ELEMENTS).disown());
}
typedef _::IndexingIterator<Builder, typename T::Builder> Iterator;
......@@ -512,7 +515,7 @@ struct List<T, Kind::BLOB> {
private:
inline static _::ListBuilder initPointer(_::PointerBuilder builder, uint size) {
return builder.initList(ElementSize::POINTER, size * ELEMENTS);
return builder.initList(ElementSize::POINTER, guarded(size) * ELEMENTS);
}
inline static _::ListBuilder getFromPointer(_::PointerBuilder builder, const word* defaultValue) {
return builder.getList(ElementSize::POINTER, defaultValue);
......
c++/src/capnp/message.c++
View file @ e2cbc1b5
......@@ -135,7 +135,7 @@ _::SegmentBuilder* MessageBuilder::getRootSegment() {
KJ_ASSERT(allocation.segment->getSegmentId() == _::SegmentId(0),
"First allocated word of new arena was not in segment ID 0.");
KJ_ASSERT(allocation.words == allocation.segment->getPtrUnchecked(0 * WORDS),
KJ_ASSERT(allocation.words == allocation.segment->getPtrUnchecked(ZERO * WORDS),
"First allocated word of new arena was not the first word in its segment.");
return allocation.segment;
}
......@@ -144,7 +144,7 @@ _::SegmentBuilder* MessageBuilder::getRootSegment() {
AnyPointer::Builder MessageBuilder::getRootInternal() {
_::SegmentBuilder* rootSegment = getRootSegment();
return AnyPointer::Builder(_::PointerBuilder::getRoot(
rootSegment, arena()->getLocalCapTable(), rootSegment->getPtrUnchecked(0 * WORDS)));
rootSegment, arena()->getLocalCapTable(), rootSegment->getPtrUnchecked(ZERO * WORDS)));
}
kj::ArrayPtr<const kj::ArrayPtr<const word>> MessageBuilder::getSegmentsForOutput() {
......
c++/src/capnp/orphan.h
View file @ e2cbc1b5
......@@ -307,17 +307,17 @@ inline ReaderFor<T> Orphan<T>::getReader() const {
template <typename T>
inline void Orphan<T>::truncate(uint size) {
_::OrphanGetImpl<ListElementType<T>>::truncateListOf(builder, size * ELEMENTS);
_::OrphanGetImpl<ListElementType<T>>::truncateListOf(builder, guarded(size) * ELEMENTS);
}
template <>
inline void Orphan<Text>::truncate(uint size) {
builder.truncateText(size * ELEMENTS);
builder.truncateText(guarded(size) * ELEMENTS);
}
template <>
inline void Orphan<Data>::truncate(uint size) {
builder.truncate(size * ELEMENTS, ElementSize::BYTE);
builder.truncate(guarded(size) * ELEMENTS, ElementSize::BYTE);
}
template <typename T>
......@@ -350,7 +350,7 @@ struct Orphanage::NewOrphanListImpl<List<T, k>> {
static inline _::OrphanBuilder apply(
_::BuilderArena* arena, _::CapTableBuilder* capTable, uint size) {
return _::OrphanBuilder::initList(
arena, capTable, size * ELEMENTS, _::ElementSizeForType<T>::value);
arena, capTable, guarded(size) * ELEMENTS, _::ElementSizeForType<T>::value);
}
};
......@@ -359,7 +359,7 @@ struct Orphanage::NewOrphanListImpl<List<T, Kind::STRUCT>> {
static inline _::OrphanBuilder apply(
_::BuilderArena* arena, _::CapTableBuilder* capTable, uint size) {
return _::OrphanBuilder::initStructList(
arena, capTable, size * ELEMENTS, _::structSize<T>());
arena, capTable, guarded(size) * ELEMENTS, _::structSize<T>());
}
};
......@@ -367,7 +367,7 @@ template <>
struct Orphanage::NewOrphanListImpl<Text> {
static inline _::OrphanBuilder apply(
_::BuilderArena* arena, _::CapTableBuilder* capTable, uint size) {
return _::OrphanBuilder::initText(arena, capTable, size * BYTES);
return _::OrphanBuilder::initText(arena, capTable, guarded(size) * BYTES);
}
};
......@@ -375,7 +375,7 @@ template <>
struct Orphanage::NewOrphanListImpl<Data> {
static inline _::OrphanBuilder apply(
_::BuilderArena* arena, _::CapTableBuilder* capTable, uint size) {
return _::OrphanBuilder::initData(arena, capTable, size * BYTES);
return _::OrphanBuilder::initData(arena, capTable, guarded(size) * BYTES);
}
};
......
c++/src/capnp/pointer-helpers.h
View file @ e2cbc1b5
......@@ -113,12 +113,12 @@ struct PointerHelpers<T, Kind::BLOB> {
static inline typename T::Reader get(PointerReader reader,
const void* defaultValue = nullptr,
uint defaultBytes = 0) {
return reader.getBlob<T>(defaultValue, defaultBytes * BYTES);
return reader.getBlob<T>(defaultValue, guarded(defaultBytes) * BYTES);
}
static inline typename T::Builder get(PointerBuilder builder,
const void* defaultValue = nullptr,
uint defaultBytes = 0) {
return builder.getBlob<T>(defaultValue, defaultBytes * BYTES);
return builder.getBlob<T>(defaultValue, guarded(defaultBytes) * BYTES);
}
static inline void set(PointerBuilder builder, typename T::Reader value) {
builder.setBlob<T>(value);
......@@ -127,7 +127,7 @@ struct PointerHelpers<T, Kind::BLOB> {
builder.setBlob<T>(value);
}
static inline typename T::Builder init(PointerBuilder builder, uint size) {
return builder.initBlob<T>(size * BYTES);
return builder.initBlob<T>(guarded(size) * BYTES);
}
static inline void adopt(PointerBuilder builder, Orphan<T>&& value) {
builder.adopt(kj::mv(value.builder));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment