Allow --packed and --flat to be used together.

In Sandstorm, we are encoding powerbox queries in packed base64 strings which may be placed in URL query parameters or the like. The strings are provided in interfaces called browser-side. We anticipate that some developers will prefer to specify a hardcoded string rather than generate it on-the-fly, since Cap'n Proto is not well-supported in browsers today, and anyway the developer may have no other reason to have a Cap'n Proto dependency at all, and powerbox queries are often static.

In this context, speed is irrelevant, while having a compact encoding is desirable. It felt sad to me to leave in the segment table in this context, adding redundant bytes when we want a compact encoding.

c++/src/capnp/compiler/capnp-test.sh

@@ -41,11 +41,13 @@ TESTDATA=`dirname "$0"`/../testdata
 $CAPNP encode $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/binary - || fail encode
 $CAPNP encode --flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/flat - || fail encode flat
 $CAPNP encode --packed $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packed - || fail encode packed
+$CAPNP encode --packed --flat $SCHEMA TestAllTypes < $TESTDATA/short.txt | cmp $TESTDATA/packedflat - || fail encode packedflat
 $CAPNP encode $SCHEMA TestAllTypes < $TESTDATA/pretty.txt | cmp $TESTDATA/binary - || fail parse pretty
 
 $CAPNP decode $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/pretty.txt - || fail decode
 $CAPNP decode --flat $SCHEMA TestAllTypes < $TESTDATA/flat | cmp $TESTDATA/pretty.txt - || fail decode flat
 $CAPNP decode --packed $SCHEMA TestAllTypes < $TESTDATA/packed | cmp $TESTDATA/pretty.txt - || fail decode packed
+$CAPNP decode --packed --flat $SCHEMA TestAllTypes < $TESTDATA/packedflat | cmp $TESTDATA/pretty.txt - || fail decode packedflat
 $CAPNP decode --short $SCHEMA TestAllTypes < $TESTDATA/binary | cmp $TESTDATA/short.txt - || fail decode short
 
 $CAPNP decode $SCHEMA TestAllTypes < $TESTDATA/segmented | cmp $TESTDATA/pretty.txt - || fail decode segmented

c++/src/capnp/compiler/capnp.c++

@@ -593,13 +593,11 @@ public:
   }
   kj::MainBuilder::Validity codeFlat() {
     if (binary) return "cannot be used with --binary";
-    if (packed) return "cannot be used with --packed";
     flat = true;
     return true;
   }
   kj::MainBuilder::Validity codePacked() {
     if (binary) return "cannot be used with --binary";
-    if (flat) return "cannot be used with --flat";
     packed = true;
     return true;
   }
@@ -683,13 +681,22 @@ public:
           input.skip(buffer.size());
         }
 
+        if (packed) {
+          words = kj::heapArray<word>(computeUnpackedSizeInWords(allBytes));
+          kj::ArrayInputStream input(allBytes);
+          capnp::_::PackedInputStream unpacker(input);
+          unpacker.read(words.asBytes().begin(), words.asBytes().size());
+          word dummy;
+          KJ_ASSERT(unpacker.tryRead(&dummy, sizeof(dummy), sizeof(dummy)) == 0);
+        } else {
           // Technically we don't know if the bytes are aligned so we'd better copy them to a new
-        // array.  Note that if we have a non-whole number of words we chop off the straggler bytes.
-        // This is fine because if those bytes are actually part of the message we will hit an error
-        // later and if they are not then who cares?
+          // array. Note that if we have a non-whole number of words we chop off the straggler
+          // bytes. This is fine because if those bytes are actually part of the message we will
+          // hit an error later and if they are not then who cares?
           words = kj::heapArray<word>(allBytes.size() / sizeof(word));
           memcpy(words.begin(), allBytes.begin(), words.size() * sizeof(word));
         }
+      }
 
       kj::ArrayPtr<const word> segments = words;
       decodeInner<SegmentArrayMessageReader>(arrayPtr(&segments, 1));
@@ -869,7 +876,8 @@ private:
     return isPlausiblyFlat(prefix.slice(segment0Offset, prefix.size()), segmentCount);
   }
 
-  Plausibility isPlausiblyPacked(kj::ArrayPtr<const byte> prefix) {
+  Plausibility isPlausiblyPacked(kj::ArrayPtr<const byte> prefix,
+      kj::Function<Plausibility(kj::ArrayPtr<const byte>)> checkUnpacked) {
     kj::Vector<byte> unpacked;
 
     // Try to unpack a prefix so that we can check it.
@@ -910,11 +918,93 @@ private:
       }
     }
 
-    return isPlausiblyBinary(unpacked);
+    return checkUnpacked(unpacked);
+  }
+
+  Plausibility isPlausiblyPacked(kj::ArrayPtr<const byte> prefix) {
+    return isPlausiblyPacked(prefix, KJ_BIND_METHOD(*this, isPlausiblyBinary));
+  }
+
+  Plausibility isPlausiblyPackedFlat(kj::ArrayPtr<const byte> prefix) {
+    return isPlausiblyPacked(prefix, [this](kj::ArrayPtr<const byte> prefix) {
+      return isPlausiblyFlat(prefix);
+    });
   }
 
   kj::MainBuilder::Validity checkPlausibility(kj::ArrayPtr<const byte> prefix) {
-    if (flat) {
+    if (flat && packed) {
+      switch (isPlausiblyPackedFlat(prefix)) {
+        case PLAUSIBLE:
+          break;
+        case IMPOSSIBLE:
+          if (plausibleOrWrongType(isPlausiblyPacked(prefix))) {
+            return "The input is not in --packed --flat format.  It looks like it is in --packed "
+                   "format.  Try removing --flat.";
+          } else if (plausibleOrWrongType(isPlausiblyFlat(prefix))) {
+            return "The input is not in --packed --flat format.  It looks like it is in --flat "
+                   "format.  Try removing --packed.";
+          } else if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
+            return "The input is not in --packed --flat format.  It looks like it is in regular "
+                   "binary format.  Try removing the --packed and --flat flags.";
+          } else {
+            return "The input is not a Cap'n Proto message.";
+          }
+        case IMPLAUSIBLE:
+          if (plausibleOrWrongType(isPlausiblyPacked(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in --packed --flat format.  It looks like\n"
+                "it may be in --packed format.  I'll try to parse it in --packed --flat format\n"
+                "as you requested, but if it doesn't work, try removing --flat.  Use --quiet to\n"
+                "suppress this warning.\n"
+                "*** END WARNING ***\n");
+          } else if (plausibleOrWrongType(isPlausiblyFlat(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in --packed --flat format.  It looks like\n"
+                "it may be in --flat format.  I'll try to parse it in --packed --flat format as\n"
+                "you requested, but if it doesn't work, try removing --packed.  Use --quiet to\n"
+                "suppress this warning.\n"
+                "*** END WARNING ***\n");
+          } else if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in --packed --flat format.  It looks like\n"
+                "it may be in regular binary format.  I'll try to parse it in --packed --flat\n"
+                "format as you requested, but if it doesn't work, try removing --packed and\n"
+                "--flat.  Use --quiet to suppress this warning.\n"
+                "*** END WARNING ***\n");
+          } else {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be a Cap'n Proto message in any known\n"
+                "binary format.  I'll try to parse it anyway, but if it doesn't work, please\n"
+                "check your input.  Use --quiet to suppress this warning.\n"
+                "*** END WARNING ***\n");
+          }
+          break;
+        case WRONG_TYPE:
+          if (plausibleOrWrongType(isPlausiblyPacked(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be the type that you specified.  I'll try\n"
+                "to parse it anyway, but if it doesn't look right, please verify that you\n"
+                "have the right type.  This could also be because the input is not in --flat\n"
+                "format; indeed, it looks like this input may be in regular --packed format,\n"
+                "so you might want to try removing --flat.  Use --quiet to suppress this\n"
+                "warning.\n"
+                "*** END WARNING ***\n");
+          } else {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be the type that you specified.  I'll try\n"
+                "to parse it anyway, but if it doesn't look right, please verify that you\n"
+                "have the right type.  Use --quiet to suppress this warning.\n"
+                "*** END WARNING ***\n");
+          }
+          break;
+      }
+    } else if (flat) {
       switch (isPlausiblyFlat(prefix)) {
         case PLAUSIBLE:
           break;
@@ -922,6 +1012,9 @@ private:
           if (plausibleOrWrongType(isPlausiblyPacked(prefix))) {
             return "The input is not in --flat format.  It looks like it is in --packed format.  "
                    "Try that instead.";
+          } else if (plausibleOrWrongType(isPlausiblyPackedFlat(prefix))) {
+            return "The input is not in --flat format.  It looks like it is in --packed --flat "
+                   "format.  Try adding --packed.";
           } else if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
             return "The input is not in --flat format.  It looks like it is in regular binary "
                    "format.  Try removing the --flat flag.";
@@ -937,6 +1030,14 @@ private:
                 "requested, but if it doesn't work, try --packed instead.  Use --quiet to\n"
                 "suppress this warning.\n"
                 "*** END WARNING ***\n");
+          } else if (plausibleOrWrongType(isPlausiblyPackedFlat(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in --flat format.  It looks like it may\n"
+                "be in --packed --flat format.  I'll try to parse it in --flat format as you\n"
+                "requested, but if it doesn't work, try adding --packed.  Use --quiet to\n"
+                "suppress this warning.\n"
+                "*** END WARNING ***\n");
           } else if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
             context.warning(
                 "*** WARNING ***\n"
@@ -983,6 +1084,10 @@ private:
           if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
             return "The input is not in --packed format.  It looks like it is in regular binary "
                    "format.  Try removing the --packed flag.";
+          } else if (plausibleOrWrongType(isPlausiblyPackedFlat(prefix))) {
+            return "The input is not in --packed format, nor does it look like it is in regular "
+                   "binary format.  It looks like it could be in --packed --flat format, although "
+                   "that is unusual so I could be wrong.";
           } else if (plausibleOrWrongType(isPlausiblyFlat(prefix))) {
             return "The input is not in --packed format, nor does it look like it is in regular "
                    "binary format.  It looks like it could be in --flat format, although that "
@@ -991,7 +1096,15 @@ private:
             return "The input is not a Cap'n Proto message.";
           }
         case IMPLAUSIBLE:
-          if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
+          if (plausibleOrWrongType(isPlausiblyPackedFlat(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in --packed format.  It looks like it may\n"
+                "be in --packed --flat format.  I'll try to parse it in --packed format as you\n"
+                "requested, but if it doesn't work, try adding --flat.  Use --quiet to\n"
+                "suppress this warning.\n"
+                "*** END WARNING ***\n");
+          } else if (plausibleOrWrongType(isPlausiblyBinary(prefix))) {
             context.warning(
                 "*** WARNING ***\n"
                 "The input data does not appear to be in --packed format.  It looks like it\n"
@@ -1039,6 +1152,10 @@ private:
             return "The input is not in regular binary format, nor does it look like it is in "
                    "--packed format.  It looks like it could be in --flat format, although that "
                    "is unusual so I could be wrong.";
+          } else if (plausibleOrWrongType(isPlausiblyPackedFlat(prefix))) {
+            return "The input is not in regular binary format, nor does it look like it is in "
+                   "--packed format.  It looks like it could be in --packed --flat format, "
+                   "although that is unusual so I could be wrong.";
           } else {
             return "The input is not a Cap'n Proto message.";
           }
@@ -1051,6 +1168,14 @@ private:
                 "requested, but if it doesn't work, try adding --packed.  Use --quiet to\n"
                 "suppress this warning.\n"
                 "*** END WARNING ***\n");
+          } else if (plausibleOrWrongType(isPlausiblyPacked(prefix))) {
+            context.warning(
+                "*** WARNING ***\n"
+                "The input data does not appear to be in regular binary format.  It looks like\n"
+                "it may be in --packed --flat format.  I'll try to parse it in regular format as\n"
+                "you requested, but if it doesn't work, try adding --packed --flat.  Use --quiet\n"
+                "to suppress this warning.\n"
+                "*** END WARNING ***\n");
           } else if (plausibleOrWrongType(isPlausiblyFlat(prefix))) {
             context.warning(
                 "*** WARNING ***\n"
@@ -1313,7 +1438,10 @@ private:
     flatMessage.setRoot(value);
     flatMessage.requireFilled();
 
-    if (flat) {
+    if (flat && packed) {
+      capnp::_::PackedOutputStream packer(output);
+      packer.write(space.begin(), space.size() * sizeof(word));
+    } else if (flat) {
       output.write(space.begin(), space.size() * sizeof(word));
     } else if (packed) {
       writePackedMessage(output, flatMessage);

c++/src/capnp/serialize-packed-test.c++

@@ -39,6 +39,11 @@ public:
   ~TestPipe() {}
 
   const std::string& getData() { return data; }
+
+  kj::ArrayPtr<const byte> getArray() {
+    return kj::arrayPtr(reinterpret_cast<const byte*>(data.data()), data.size());
+  }
+
   void resetRead(size_t preferredReadSize = kj::maxValue) {
     readPos = 0;
     this->preferredReadSize = preferredReadSize;
@@ -84,6 +89,8 @@ private:
 void expectPacksTo(kj::ArrayPtr<const byte> unpacked, kj::ArrayPtr<const byte> packed) {
   TestPipe pipe;
 
+  EXPECT_EQ(unpacked.size(), computeUnpackedSizeInWords(packed) * sizeof(word));
+
   // -----------------------------------------------------------------
   // write
 
@@ -244,6 +251,8 @@ TEST(Packed, RoundTrip) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -255,6 +264,8 @@ TEST(Packed, RoundTripScratchSpace) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   word scratch[1024];
   PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
   checkTestMessage(reader.getRoot<TestAllTypes>());
@@ -267,6 +278,8 @@ TEST(Packed, RoundTripLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -278,6 +291,8 @@ TEST(Packed, RoundTripOddSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -289,6 +304,8 @@ TEST(Packed, RoundTripOddSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -300,6 +317,8 @@ TEST(Packed, RoundTripEvenSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -311,6 +330,8 @@ TEST(Packed, RoundTripEvenSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessage(reader.getRoot<TestAllTypes>());
 }
@@ -326,6 +347,9 @@ TEST(Packed, RoundTripTwoMessages) {
   writePackedMessage(pipe, builder);
   writePackedMessage(pipe, builder2);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder) + computeSerializedSizeInWords(builder2),
+            computeUnpackedSizeInWords(pipe.getArray()));
+
   {
     PackedMessageReader reader(pipe);
     checkTestMessage(reader.getRoot<TestAllTypes>());
@@ -346,6 +370,8 @@ TEST(Packed, RoundTripAllZero) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 
@@ -362,6 +388,8 @@ TEST(Packed, RoundTripAllZeroScratchSpace) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   word scratch[1024];
   PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
@@ -374,6 +402,8 @@ TEST(Packed, RoundTripAllZeroLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 }
@@ -385,6 +415,8 @@ TEST(Packed, RoundTripAllZeroOddSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 }
@@ -396,6 +428,8 @@ TEST(Packed, RoundTripAllZeroOddSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 }
@@ -407,6 +441,8 @@ TEST(Packed, RoundTripAllZeroEvenSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 }
@@ -418,6 +454,8 @@ TEST(Packed, RoundTripAllZeroEvenSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   checkTestMessageAllZero(reader.getRoot<TestAllTypes>());
 }
@@ -434,6 +472,8 @@ TEST(Packed, RoundTripHugeString) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }
@@ -448,6 +488,8 @@ TEST(Packed, RoundTripHugeStringScratchSpace) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   word scratch[1024];
   PackedMessageReader reader(pipe, ReaderOptions(), kj::ArrayPtr<word>(scratch, 1024));
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
@@ -463,6 +505,8 @@ TEST(Packed, RoundTripHugeStringLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }
@@ -477,6 +521,8 @@ TEST(Packed, RoundTripHugeStringOddSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }
@@ -491,6 +537,8 @@ TEST(Packed, RoundTripHugeStringOddSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }
@@ -505,6 +553,8 @@ TEST(Packed, RoundTripHugeStringEvenSegmentCount) {
   TestPipe pipe;
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }
@@ -519,6 +569,8 @@ TEST(Packed, RoundTripHugeStringEvenSegmentCountLazy) {
   TestPipe pipe(1);
   writePackedMessage(pipe, builder);
 
+  EXPECT_EQ(computeSerializedSizeInWords(builder), computeUnpackedSizeInWords(pipe.getArray()));
+
   PackedMessageReader reader(pipe);
   EXPECT_TRUE(reader.getRoot<TestAllTypes>().getTextField() == huge);
 }

c++/src/capnp/serialize-packed.c++

@@ -43,7 +43,7 @@ size_t PackedInputStream::tryRead(void* dst, size_t minBytes, size_t maxBytes) {
   uint8_t* const outEnd = reinterpret_cast<uint8_t*>(dst) + maxBytes;
   uint8_t* const outMin = reinterpret_cast<uint8_t*>(dst) + minBytes;
 
-  kj::ArrayPtr<const byte> buffer = inner.getReadBuffer();
+  kj::ArrayPtr<const byte> buffer = inner.tryGetReadBuffer();
   if (buffer.size() == 0) {
     return 0;
   }
@@ -476,4 +476,32 @@ void writePackedMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>>
   writePackedMessage(output, segments);
 }
 
+size_t computeUnpackedSizeInWords(kj::ArrayPtr<const byte> packedBytes) {
+  const byte* ptr = packedBytes.begin();
+  const byte* end = packedBytes.end();
+
+  size_t total = 0;
+  while (ptr < end) {
+    uint tag = *ptr;
+    size_t count = __builtin_popcount(tag);
+    total += 1;
+    KJ_REQUIRE(end - ptr >= count, "invalid packed data");
+    ptr += count + 1;
+
+    if (tag == 0) {
+      KJ_REQUIRE(ptr < end, "invalid packed data");
+      total += *ptr++;
+    } else if (tag == 0xff) {
+      KJ_REQUIRE(ptr < end, "invalid packed data");
+      size_t words = *ptr++;
+      total += words;
+      size_t bytes = words * sizeof(word);
+      KJ_REQUIRE(end - ptr >= bytes, "invalid packed data");
+      ptr += bytes;
+    }
+  }
+
+  return total;
+}
+
 }  // namespace capnp

c++/src/capnp/serialize-packed.h

@@ -106,6 +106,10 @@ void writePackedMessageToFd(int fd, MessageBuilder& builder);
 void writePackedMessageToFd(int fd, kj::ArrayPtr<const kj::ArrayPtr<const word>> segments);
 // Write a single packed message to the file descriptor.
 
+size_t computeUnpackedSizeInWords(kj::ArrayPtr<const byte> packedBytes);
+// Computes the number of words to which the given packed bytes will unpack. Not intended for use
+// in performance-sensitive situations.
+
 // =======================================================================================
 // inline stuff