Skip to content

C
capnproto
Commit 4361912b authored by Kenton Varda's avatar Kenton Varda
Browse files
Options

Allow --packed and --flat to be used together. 

In Sandstorm, we are encoding powerbox queries in packed base64 strings which may be placed in URL query parameters or the like. The strings are provided in interfaces called browser-side. We anticipate that some developers will prefer to specify a hardcoded string rather than generate it on-the-fly, since Cap'n Proto is not well-supported in browsers today, and anyway the developer may have no other reason to have a Cap'n Proto dependency at all, and powerbox queries are often static.

In this context, speed is irrelevant, while having a compact encoding is desirable. It felt sad to me to leave in the segment table in this context, adding redundant bytes when we want a compact encoding.
parent c242a3a0
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 87 additions and 1 deletion
c++/src/capnp/compiler/capnp-test.sh
View file @ 4361912b
......@@ -41,11 +41,13 @@ TESTDATA=`dirname "$0"`/../testdata
$CAPNP encode $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/binary - || fail encode
$CAPNP encode --flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/flat - || fail encode flat
$CAPNP encode --packed $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packed - || fail encode packed
$CAPNP encode --packed --flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packedflat - || fail encode packedflat
$CAPNP encode $SCHEMA TestAllTypes < $TESTDATA/pretty.txt | cmp $TESTDATA/binary - || fail parse pretty
$CAPNP decode $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/pretty.txt - || fail decode
$CAPNP decode --flat $SCHEMA TestAllTypes < $TESTDATA/flat | cmp $TESTDATA/pretty.txt - || fail decode flat
$CAPNP decode --packed $SCHEMA TestAllTypes < $TESTDATA/packed | cmp $TESTDATA/pretty.txt - || fail decode packed
$CAPNP decode --packed --flat $SCHEMA TestAllTypes < $TESTDATA/packedflat | cmp $TESTDATA/pretty.txt - || fail decode packedflat
$CAPNP decode --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/short.txt - || fail decode short
$CAPNP decode $SCHEMA TestAllTypes < $TESTDATA/segmented | cmp $TESTDATA/pretty.txt - || fail decode segmented
......
c++/src/capnp/compiler/capnp.c++
View file @ 4361912b
This diff is collapsed.
c++/src/capnp/serialize-packed-test.c++
View file @ 4361912b
......@@ -39,6 +39,11 @@ public:
~TestPipe() {}
const std::string& getData() { return data; }
kj::ArrayPtr<const byte> getArray() {
return kj::arrayPtr(reinterpret_cast<const byte*>(data.data()), data.size());
}
void resetRead(size_t preferredReadSize = kj::maxValue) {
readPos = 0;
this->preferredReadSize = preferredReadSize;
......@@ -84,6 +89,8 @@ private:
void expectPacksTo(kj::ArrayPtr<const byte> unpacked, kj::ArrayPtr<const byte> packed) {
TestPipe pipe;
EXPECT_EQ(unpacked.size(), computeUnpackedSizeInWords(packed) * sizeof(word));
// -----------------------------------------------------------------
// write
......@@ -244,6 +251,8 @@ TEST(Packed, RoundTrip) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -255,6 +264,8 @@ TEST(Packed, RoundTripScratchSpace) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
word scratch[1024];
PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
checkTestMessage(reader.getRoot<TestAllTypes>());
......@@ -267,6 +278,8 @@ TEST(Packed, RoundTripLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -278,6 +291,8 @@ TEST(Packed, RoundTripOddSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -289,6 +304,8 @@ TEST(Packed, RoundTripOddSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -300,6 +317,8 @@ TEST(Packed, RoundTripEvenSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -311,6 +330,8 @@ TEST(Packed, RoundTripEvenSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
}
......@@ -326,6 +347,9 @@ TEST(Packed, RoundTripTwoMessages) {
writePackedMessage(pipe, builder);
writePackedMessage(pipe, builder2);
EXPECT_EQ(computeSerializedSizeInWords(builder) + computeSerializedSizeInWords(builder2),
computeUnpackedSizeInWords(pipe.getArray()));
{
PackedMessageReader reader(pipe);
checkTestMessage(reader.getRoot<TestAllTypes>());
......@@ -346,6 +370,8 @@ TEST(Packed, RoundTripAllZero) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
......@@ -362,6 +388,8 @@ TEST(Packed, RoundTripAllZeroScratchSpace) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
word scratch[1024];
PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
......@@ -374,6 +402,8 @@ TEST(Packed, RoundTripAllZeroLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
}
......@@ -385,6 +415,8 @@ TEST(Packed, RoundTripAllZeroOddSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
}
......@@ -396,6 +428,8 @@ TEST(Packed, RoundTripAllZeroOddSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
}
......@@ -407,6 +441,8 @@ TEST(Packed, RoundTripAllZeroEvenSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
}
......@@ -418,6 +454,8 @@ TEST(Packed, RoundTripAllZeroEvenSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
}
......@@ -434,6 +472,8 @@ TEST(Packed, RoundTripHugeString) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......@@ -448,6 +488,8 @@ TEST(Packed, RoundTripHugeStringScratchSpace) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
word scratch[1024];
PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
......@@ -463,6 +505,8 @@ TEST(Packed, RoundTripHugeStringLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......@@ -477,6 +521,8 @@ TEST(Packed, RoundTripHugeStringOddSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......@@ -491,6 +537,8 @@ TEST(Packed, RoundTripHugeStringOddSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......@@ -505,6 +553,8 @@ TEST(Packed, RoundTripHugeStringEvenSegmentCount) {
TestPipe pipe;
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......@@ -519,6 +569,8 @@ TEST(Packed, RoundTripHugeStringEvenSegmentCountLazy) {
TestPipe pipe(1);
writePackedMessage(pipe, builder);
EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
PackedMessageReader reader(pipe);
EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
}
......
c++/src/capnp/serialize-packed.c++
View file @ 4361912b
......@@ -43,7 +43,7 @@ size_t PackedInputStream::tryRead(void* dst, size_t minBytes, size_t maxBytes) {
uint8_t* const outEnd = reinterpret_cast<uint8_t*>(dst) + maxBytes;
uint8_t* const outMin = reinterpret_cast<uint8_t*>(dst) + minBytes;
kj::ArrayPtr<const byte> buffer = inner.getReadBuffer();
kj::ArrayPtr<const byte> buffer = inner.tryGetReadBuffer();
if (buffer.size() == 0) {
return 0;
}
......@@ -476,4 +476,32 @@ void writePackedMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>>
writePackedMessage(output, segments);
}
size_t computeUnpackedSizeInWords(kj::ArrayPtr<const byte> packedBytes) {
const byte* ptr = packedBytes.begin();
const byte* end = packedBytes.end();
size_t total = 0;
while (ptr < end) {
uint tag = *ptr;
size_t count = __builtin_popcount(tag);
total += 1;
KJ_REQUIRE(end - ptr >= count, "invalid packed data");
ptr += count + 1;
if (tag == 0) {
KJ_REQUIRE(ptr < end, "invalid packed data");
total += *ptr++;
} else if (tag == 0xff) {
KJ_REQUIRE(ptr < end, "invalid packed data");
size_t words = *ptr++;
total += words;
size_t bytes = words * sizeof(word);
KJ_REQUIRE(end - ptr >= bytes, "invalid packed data");
ptr += bytes;
}
}
return total;
}
} // namespace capnp
c++/src/capnp/serialize-packed.h
View file @ 4361912b
......@@ -106,6 +106,10 @@ void writePackedMessageToFd(int fd, MessageBuilder& builder);
void writePackedMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
// Write a single packed message to the file descriptor.
size_t computeUnpackedSizeInWords(kj::ArrayPtr<const byte> packedBytes);
// Computes the number of words to which the given packed bytes will unpack. Not intended for use
// in performance-sensitive situations.
// =======================================================================================
// inline stuff
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment