Skip to content

C
capnproto
Commit 79b7f287 authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Allow far references to be in a segment separate from either the source or target.

parent 952ae5c5
Show whitespace changes
Inline Side-by-side
Showing with 150 additions and 102 deletions
c++/src/capnproto/wire-format.c++
View file @ 79b7f287
......@@ -64,6 +64,13 @@ struct WireReference {
inline WordCount wordSize() const {
return dataSize.get() + refCount.get() * WORDS_PER_REFERENCE;
}
CAPNPROTO_ALWAYS_INLINE(void set(FieldNumber fc, WordCount ds, WireReferenceCount rc)) {
fieldCount.set(fc);
dataSize.set(ds);
refCount.set(rc);
reserved0.set(0);
}
} structRef;
// Also covers capabilities.
......@@ -79,6 +86,19 @@ struct WireReference {
CAPNPROTO_ALWAYS_INLINE(WordCount inlineCompositeWordCount() const) {
return elementCount() * (1 * WORDS / ELEMENTS);
}
CAPNPROTO_ALWAYS_INLINE(void set(FieldSize es, ElementCount ec)) {
CAPNPROTO_DEBUG_ASSERT(ec < (1 << 29) * ELEMENTS,
"Lists are limited to 2**29 elements.");
elementSizeAndCount.set((static_cast<int>(es) << 29) | (ec / ELEMENTS));
}
CAPNPROTO_ALWAYS_INLINE(void setInlineComposite(WordCount wc)) {
CAPNPROTO_DEBUG_ASSERT(wc < (1 << 29) * WORDS,
"Inline composite lists are limited to 2**29 words.");
elementSizeAndCount.set(
(static_cast<int>(FieldSize::INLINE_COMPOSITE) << 29) | (wc / WORDS));
}
} listRef;
struct {
......@@ -111,54 +131,39 @@ struct WireReference {
offsetAndTag.set(((elementCount / ELEMENTS) << 3) | tag);
}
CAPNPROTO_ALWAYS_INLINE(void setStruct(
FieldNumber fieldCount, WordCount dataSize, WireReferenceCount refCount, word* target)) {
setTagAndOffset(STRUCT, intervalLength(reinterpret_cast<word*>(this), target));
structRef.fieldCount.set(fieldCount);
structRef.dataSize.set(WordCount8(dataSize));
structRef.refCount.set(refCount);
structRef.reserved0.set(0);
}
// CAPNPROTO_ALWAYS_INLINE(void setStruct(
// FieldNumber fieldCount, WordCount dataSize, WireReferenceCount refCount, word* target)) {
// setTagAndOffset(STRUCT, intervalLength(reinterpret_cast<word*>(this), target));
// structRef.set(fieldCount, dataSize, refCount);
// }
//
CAPNPROTO_ALWAYS_INLINE(void setStructTag(
FieldNumber fieldCount, WordCount dataSize, WireReferenceCount refCount,
ElementCount elementCount)) {
setTagAndElementCount(STRUCT, elementCount);
structRef.fieldCount.set(fieldCount);
structRef.dataSize.set(WordCount8(dataSize));
structRef.refCount.set(refCount);
structRef.reserved0.set(0);
}
CAPNPROTO_ALWAYS_INLINE(void setList(
FieldSize elementSize, ElementCount elementCount, word* target)) {
setTagAndOffset(LIST, intervalLength(reinterpret_cast<word*>(this), target));
CAPNPROTO_DEBUG_ASSERT(elementCount < (1 << 29) * ELEMENTS,
"Lists are limited to 2**29 elements.");
listRef.elementSizeAndCount.set(
(static_cast<int>(elementSize) << 29) | (elementCount / ELEMENTS));
}
CAPNPROTO_ALWAYS_INLINE(void setEmptyList(FieldSize elementSize)) {
setTagAndOffset(LIST, 0 * WORDS);
listRef.elementSizeAndCount.set(static_cast<int>(elementSize) << 29);
}
CAPNPROTO_ALWAYS_INLINE(void setInlineCompositeList(WordCount wordCount, word* target)) {
setTagAndOffset(LIST, intervalLength(reinterpret_cast<word*>(this), target));
CAPNPROTO_DEBUG_ASSERT(wordCount < (1 << 29) * WORDS,
"Inline composite lists are limited to 2**29 words.");
listRef.elementSizeAndCount.set(
(static_cast<int>(FieldSize::INLINE_COMPOSITE) << 29) | (wordCount / WORDS));
structRef.set(fieldCount, dataSize, refCount);
}
// CAPNPROTO_ALWAYS_INLINE(void setList(
// FieldSize elementSize, ElementCount elementCount, word* target)) {
// setTagAndOffset(LIST, intervalLength(reinterpret_cast<word*>(this), target));
// listRef.set(elementSize, elementCount);
// }
//
// CAPNPROTO_ALWAYS_INLINE(void setEmptyList(FieldSize elementSize)) {
// setTagAndOffset(LIST, 0 * WORDS);
// listRef.set(elementSize, 0 * ELEMENTS);
// }
//
// CAPNPROTO_ALWAYS_INLINE(void setInlineCompositeList(WordCount wordCount, word* target)) {
// setTagAndOffset(LIST, intervalLength(reinterpret_cast<word*>(this), target));
// listRef.setInlineComposite(wordCount);
// }
//
CAPNPROTO_ALWAYS_INLINE(void setListTag(FieldSize elementSize, ElementCount listCount,
ElementCount elementsPerList)) {
setTagAndElementCount(LIST, listCount);
CAPNPROTO_DEBUG_ASSERT(elementsPerList < (1 << 29) * ELEMENTS,
"Lists are limited to 2**29 elements.");
listRef.elementSizeAndCount.set(
(static_cast<int>(elementSize) << 29) | (elementsPerList / ELEMENTS));
listRef.set(elementSize, elementsPerList);
}
CAPNPROTO_ALWAYS_INLINE(void setFar(SegmentId segmentId, WordCount offset)) {
......@@ -185,54 +190,89 @@ struct WireHelpers {
}
static CAPNPROTO_ALWAYS_INLINE(word* allocate(
WireReference*& ref, SegmentBuilder*& segment, WordCount amount)) {
WireReference*& ref, SegmentBuilder*& segment, WordCount amount,
WireReference::Tag tag)) {
word* ptr = segment->allocate(amount);
if (ptr == nullptr) {
// Need to allocate in a new segment.
// Need to allocate in a new segment. We'll need to allocate an extra reference worth of
// space to act as the landing pad for a far reference.
// Loop here just in case we ever make Segment::allocate() thread-safe -- in this case another
// thread could have grabbed the space between when we asked the message for the segment and
// when we asked the segment to allocate space.
do {
WordCount amountPlusRef = amount + REFERENCE_SIZE_IN_WORDS;
segment = segment->getMessage()->getSegmentWithAvailable(amountPlusRef);
ptr = segment->allocate(amountPlusRef);
} while (CAPNPROTO_EXPECT_FALSE(ptr == nullptr));
// Set up the original reference to be a far reference to the new segment.
ref->setFar(segment->getSegmentId(), segment->getOffsetTo(ptr));
// The landing pad has an offset of zero to indicate that the data immediately follows it.
ref = reinterpret_cast<WireReference*>(ptr);
ref->setTagAndOffset(tag, 0 * WORDS);
// Allocated space follows new reference.
return ptr + REFERENCE_SIZE_IN_WORDS;
} else {
ref->setTagAndOffset(tag, intervalLength(reinterpret_cast<word*>(ref), ptr));
return ptr;
}
}
static CAPNPROTO_ALWAYS_INLINE(void followFars(WireReference*& ref, SegmentBuilder*& segment)) {
static CAPNPROTO_ALWAYS_INLINE(word* followFars(WireReference*& ref, SegmentBuilder*& segment)) {
if (ref->tag() == WireReference::FAR) {
segment = segment->getMessage()->getSegment(ref->farRef.segmentId.get());
ref = reinterpret_cast<WireReference*>(segment->getPtrUnchecked(ref->offset()));
if (ref->offset() == 0 * WORDS) {
// Target immediately follows tag.
return reinterpret_cast<word*>(ref + 1);
} else {
// Target is in another castle. Another far reference follows.
WireReference* far2 = ref + 1;
segment = segment->getMessage()->getSegment(far2->farRef.segmentId.get());
return segment->getPtrUnchecked(far2->offset());
}
} else {
return ref->target();
}
}
static CAPNPROTO_ALWAYS_INLINE(bool followFars(
const WireReference*& ref, SegmentReader*& segment)) {
static CAPNPROTO_ALWAYS_INLINE(
const word* followFars(const WireReference*& ref, SegmentReader*& segment)) {
if (ref->tag() == WireReference::FAR) {
segment = segment->getMessage()->tryGetSegment(ref->farRef.segmentId.get());
if (CAPNPROTO_EXPECT_FALSE(segment == nullptr)) {
return false;
return nullptr;
}
const word* ptr = segment->getStartPtr() + ref->offset();
if (CAPNPROTO_EXPECT_FALSE(!segment->containsInterval(ptr, ptr + REFERENCE_SIZE_IN_WORDS))) {
return false;
if (CAPNPROTO_EXPECT_FALSE(!segment->containsInterval(
ptr, ptr + REFERENCE_SIZE_IN_WORDS))) {
return nullptr;
}
ref = reinterpret_cast<const WireReference*>(ptr);
if (ref->offset() == 0 * WORDS) {
// Target immediately follows tag.
return reinterpret_cast<const word*>(ref + 1);
} else {
// Target is in another castle. Another far reference follows.
const WireReference* far2 = ref + 1;
segment = segment->getMessage()->tryGetSegment(far2->farRef.segmentId.get());
if (CAPNPROTO_EXPECT_FALSE(segment == nullptr)) {
return nullptr;
}
ptr = segment->getStartPtr() + far2->offset();
if (CAPNPROTO_EXPECT_FALSE(!segment->containsInterval(
ptr, ptr + REFERENCE_SIZE_IN_WORDS))) {
return nullptr;
}
return ptr;
}
} else {
return ref->target();
}
return true;
}
// -----------------------------------------------------------------
......@@ -247,28 +287,31 @@ struct WireHelpers {
uint n = referenceCount / REFERENCES;
for (uint i = 0; i < n; i++) {
copyMessage(segment, dstRefs + i, srcRefs + i);
SegmentBuilder* subSegment = segment;
WireReference* dstRef = dstRefs + i;
copyMessage(subSegment, dstRef, srcRefs + i);
}
}
// Not always-inline because it's recursive.
static WireReference* copyMessage(
SegmentBuilder* segment, WireReference* dst, const WireReference* src) {
static word* copyMessage(
SegmentBuilder*& segment, WireReference*& dst, const WireReference* src) {
switch (src->tag()) {
case WireReference::STRUCT: {
if (src->isNull()) {
memset(dst, 0, sizeof(WireReference));
return nullptr;
} else {
const word* srcPtr = src->target();
word* dstPtr = allocate(dst, segment, src->structRef.wordSize());
word* dstPtr = allocate(dst, segment, src->structRef.wordSize(), WireReference::STRUCT);
copyStruct(segment, dstPtr, srcPtr, dst->structRef.dataSize.get(),
dst->structRef.refCount.get());
dst->setStruct(dst->structRef.fieldCount.get(), dst->structRef.dataSize.get(),
dst->structRef.refCount.get(), dstPtr);
dst->structRef.set(dst->structRef.fieldCount.get(), dst->structRef.dataSize.get(),
dst->structRef.refCount.get());
return dstPtr;
}
break;
}
case WireReference::LIST: {
switch (src->listRef.elementSize()) {
......@@ -282,53 +325,56 @@ struct WireHelpers {
ElementCount64(src->listRef.elementCount()) *
bitsPerElement(src->listRef.elementSize()));
const word* srcPtr = src->target();
word* dstPtr = allocate(dst, segment, wordCount);
word* dstPtr = allocate(dst, segment, wordCount, WireReference::LIST);
memcpy(dstPtr, srcPtr, wordCount * BYTES_PER_WORD / BYTES);
dst->setList(src->listRef.elementSize(), src->listRef.elementCount(), dstPtr);
break;
dst->listRef.set(src->listRef.elementSize(), src->listRef.elementCount());
return dstPtr;
}
case FieldSize::REFERENCE: {
const WireReference* srcRefs = reinterpret_cast<const WireReference*>(src->target());
WireReference* dstRefs = reinterpret_cast<WireReference*>(
allocate(dst, segment, src->listRef.elementCount() *
(1 * REFERENCES / ELEMENTS) * WORDS_PER_REFERENCE));
(1 * REFERENCES / ELEMENTS) * WORDS_PER_REFERENCE,
WireReference::LIST));
uint n = src->listRef.elementCount() / ELEMENTS;
for (uint i = 0; i < n; i++) {
copyMessage(segment, dstRefs + i, srcRefs + i);
SegmentBuilder* subSegment = segment;
WireReference* dstRef = dstRefs + i;
copyMessage(subSegment, dstRef, srcRefs + i);
}
dst->setList(FieldSize::REFERENCE, src->listRef.elementCount(),
reinterpret_cast<word*>(dstRefs));
break;
dst->listRef.set(FieldSize::REFERENCE, src->listRef.elementCount());
return reinterpret_cast<word*>(dstRefs);
}
case FieldSize::INLINE_COMPOSITE: {
const word* srcPtr = src->target();
word* dstPtr = allocate(dst, segment,
src->listRef.inlineCompositeWordCount() + REFERENCE_SIZE_IN_WORDS);
src->listRef.inlineCompositeWordCount() + REFERENCE_SIZE_IN_WORDS,
WireReference::LIST);
dst->setInlineCompositeList(src->listRef.inlineCompositeWordCount(), dstPtr);
dst->listRef.setInlineComposite(src->listRef.inlineCompositeWordCount());
const WireReference* srcTag = reinterpret_cast<const WireReference*>(srcPtr);
memcpy(dstPtr, srcTag, sizeof(WireReference));
srcPtr += REFERENCE_SIZE_IN_WORDS;
dstPtr += REFERENCE_SIZE_IN_WORDS;
const word* srcElement = srcPtr + REFERENCE_SIZE_IN_WORDS;
word* dstElement = dstPtr + REFERENCE_SIZE_IN_WORDS;
CAPNPROTO_ASSERT(srcTag->tag() == WireReference::STRUCT,
"INLINE_COMPOSITE of lists is not yet supported.");
uint n = srcTag->tagElementCount() / ELEMENTS;
for (uint i = 0; i < n; i++) {
copyStruct(segment, dstPtr, srcPtr,
copyStruct(segment, dstElement, srcElement,
srcTag->structRef.dataSize.get(), srcTag->structRef.refCount.get());
srcPtr += srcTag->structRef.wordSize();
dstPtr += srcTag->structRef.wordSize();
srcElement += srcTag->structRef.wordSize();
dstElement += srcTag->structRef.wordSize();
}
break;
return dstPtr;
}
}
break;
......@@ -338,7 +384,7 @@ struct WireHelpers {
break;
}
return dst;
return nullptr;
}
// -----------------------------------------------------------------
......@@ -348,7 +394,7 @@ struct WireHelpers {
const WireReference* defaultRef = reinterpret_cast<const WireReference*>(typeDefaultValue);
// Allocate space for the new struct.
word* ptr = allocate(ref, segment, defaultRef->structRef.wordSize());
word* ptr = allocate(ref, segment, defaultRef->structRef.wordSize(), WireReference::STRUCT);
// Copy over the data segment from the default value. We don't have to copy the reference
// segment because it is presumed to be all-null.
......@@ -356,8 +402,8 @@ struct WireHelpers {
defaultRef->structRef.dataSize.get() * BYTES_PER_WORD / BYTES);
// Initialize the reference.
ref->setStruct(defaultRef->structRef.fieldCount.get(), defaultRef->structRef.dataSize.get(),
defaultRef->structRef.refCount.get(), ptr);
ref->structRef.set(defaultRef->structRef.fieldCount.get(), defaultRef->structRef.dataSize.get(),
defaultRef->structRef.refCount.get());
// Build the StructBuilder.
return StructBuilder(segment, ptr,
......@@ -367,11 +413,12 @@ struct WireHelpers {
static CAPNPROTO_ALWAYS_INLINE(StructBuilder getStructReference(
WireReference* ref, SegmentBuilder* segment, const word* defaultValue)) {
const WireReference* defaultRef = reinterpret_cast<const WireReference*>(defaultValue);
word* ptr;
if (ref->isNull()) {
ref = copyMessage(segment, ref, defaultRef);
ptr = copyMessage(segment, ref, defaultRef);
} else {
followFars(ref, segment);
ptr = followFars(ref, segment);
CAPNPROTO_DEBUG_ASSERT(ref->tag() == WireReference::STRUCT,
"Called getStruct{Field,Element}() but existing reference is not a struct.");
......@@ -386,7 +433,6 @@ struct WireHelpers {
"Trying to update struct with incorrect reference count.");
}
word* ptr = ref->target();
return StructBuilder(segment, ptr,
reinterpret_cast<WireReference*>(ptr + defaultRef->structRef.dataSize.get()));
}
......@@ -402,10 +448,10 @@ struct WireHelpers {
ElementCount64(elementCount) * bitsPerElement(elementSize));
// Allocate the list.
word* ptr = allocate(ref, segment, wordCount);
word* ptr = allocate(ref, segment, wordCount, WireReference::LIST);
// Initialize the reference.
ref->setList(elementSize, elementCount, ptr);
ref->listRef.set(elementSize, elementCount);
// Build the ListBuilder.
return ListBuilder(segment, ptr, elementCount);
......@@ -420,11 +466,11 @@ struct WireHelpers {
// Allocate the list, prefixed by a single WireReference.
WordCount wordCount = elementCount * wordsPerElement;
word* ptr = allocate(ref, segment, REFERENCE_SIZE_IN_WORDS + wordCount);
word* ptr = allocate(ref, segment, REFERENCE_SIZE_IN_WORDS + wordCount, WireReference::LIST);
// Initialize the reference.
// INLINE_COMPOSITE lists replace the element count with the word count.
ref->setInlineCompositeList(wordCount, ptr);
ref->listRef.setInlineComposite(wordCount);
// Initialize the list tag.
reinterpret_cast<WireReference*>(ptr)->setStructTag(
......@@ -452,14 +498,15 @@ struct WireHelpers {
static CAPNPROTO_ALWAYS_INLINE(ListBuilder getWritableListReference(
WireReference* ref, SegmentBuilder* segment, const word* defaultValue)) {
const WireReference* defaultRef = reinterpret_cast<const WireReference*>(defaultValue);
word* ptr;
if (ref->isNull()) {
if (defaultValue == nullptr) {
return ListBuilder(segment, nullptr, 0 * ELEMENTS);
}
ref = copyMessage(segment, ref, defaultRef);
ptr = copyMessage(segment, ref, defaultRef);
} else {
followFars(ref, segment);
ptr = followFars(ref, segment);
CAPNPROTO_ASSERT(ref->tag() == WireReference::LIST,
"Called getList{Field,Element}() but existing reference is not a list.");
......@@ -467,15 +514,15 @@ struct WireHelpers {
if (ref->listRef.elementSize() == FieldSize::INLINE_COMPOSITE) {
// Read the tag to get the actual element count.
WireReference* tag = reinterpret_cast<WireReference*>(ref->target());
WireReference* tag = reinterpret_cast<WireReference*>(ptr);
CAPNPROTO_ASSERT(tag->tag() == WireReference::STRUCT,
"INLINE_COMPOSITE list with non-STRUCT elements not supported.");
ElementCount elementCount = tag->tagElementCount();
// First list element is at ptr + 1 reference.
// First list element is at tag + 1 reference.
return ListBuilder(segment, reinterpret_cast<word*>(tag + 1), elementCount);
} else {
return ListBuilder(segment, ref->target(), ref->listRef.elementCount());
return ListBuilder(segment, ptr, ref->listRef.elementCount());
}
}
......@@ -496,7 +543,8 @@ struct WireHelpers {
goto useDefault;
}
if (CAPNPROTO_EXPECT_FALSE(!followFars(ref, segment))) {
ptr = followFars(ref, segment);
if (CAPNPROTO_EXPECT_FALSE(ptr == nullptr)) {
segment->getMessage()->reportInvalidData(
"Message contains out-of-bounds far reference.");
goto useDefault;
......@@ -508,8 +556,6 @@ struct WireHelpers {
goto useDefault;
}
ptr = ref->target();
if (CAPNPROTO_EXPECT_FALSE(!segment->containsInterval(ptr, ptr + ref->structRef.wordSize()))){
segment->getMessage()->reportInvalidData(
"Message contained out-of-bounds struct reference.");
......@@ -531,14 +577,15 @@ struct WireHelpers {
static CAPNPROTO_ALWAYS_INLINE(ListReader readListReference(
SegmentReader* segment, const WireReference* ref, const word* defaultValue,
FieldSize expectedElementSize, int recursionLimit)) {
const word* ptr;
if (ref == nullptr || ref->isNull()) {
useDefault:
segment = nullptr;
if (defaultValue == nullptr) {
return ListReader(nullptr, nullptr, 0 * ELEMENTS, 0 * BITS / ELEMENTS, recursionLimit - 1);
} else {
ref = reinterpret_cast<const WireReference*>(defaultValue);
}
segment = nullptr;
ref = reinterpret_cast<const WireReference*>(defaultValue);
ptr = ref->target();
} else if (segment != nullptr) {
if (CAPNPROTO_EXPECT_FALSE(recursionLimit == 0)) {
segment->getMessage()->reportInvalidData(
......@@ -546,7 +593,8 @@ struct WireHelpers {
goto useDefault;
}
if (CAPNPROTO_EXPECT_FALSE(!followFars(ref, segment))) {
ptr = followFars(ref, segment);
if (CAPNPROTO_EXPECT_FALSE(ptr == nullptr)) {
segment->getMessage()->reportInvalidData(
"Message contains out-of-bounds far reference.");
goto useDefault;
......@@ -557,13 +605,15 @@ struct WireHelpers {
"Message contains non-list reference where list reference was expected.");
goto useDefault;
}
} else {
// Trusted messages don't contain far pointers.
ptr = ref->target();
}
if (ref->listRef.elementSize() == FieldSize::INLINE_COMPOSITE) {
decltype(WORDS/ELEMENTS) wordsPerElement;
ElementCount size;
const word* ptr = ref->target();
WordCount wordCount = ref->listRef.inlineCompositeWordCount();
// An INLINE_COMPOSITE list points to a tag, which is formatted like a reference.
......@@ -652,8 +702,6 @@ struct WireHelpers {
// The elements of the list are NOT structs.
decltype(BITS/ELEMENTS) step = bitsPerElement(ref->listRef.elementSize());
const word* ptr = ref->target();
if (segment != nullptr) {
if (CAPNPROTO_EXPECT_FALSE(!segment->containsInterval(ptr, ptr +
roundUpToWords(ElementCount64(ref->listRef.elementCount()) * step)))) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment