Merge pull request #293 from maurer/canonicalize-pr

Add Canonicalization

CONTRIBUTORS

@@ -15,6 +15,7 @@ Kamal Marhubi <kamal@marhubi.com>: JSON parser
 Oliver Kuckertz <oliver.kuckertz@mologie.de>: FdObserver POLLPRI support
 Harris Hancock <vortrab@gmail.com>: MSVC support
 Branislav Katreniak <branislav.katreniak@digitalstrom.com>: JSON decode
+Matthew Maurer <matthew.r.maurer@gmail.com>: Canonicalization Support
 
 This file does not list people who maintain their own Cap'n Proto
 implementations as separate projects.  Those people are awesome too!  :)

c++/src/capnp/CMakeLists.txt

@@ -210,6 +210,7 @@ if(BUILD_TESTING)
     orphan-test.c++
     serialize-test.c++
     serialize-packed-test.c++
+    canonicalize-test.c++
     fuzz-test.c++
     test-util.c++
     ${test_capnp_cpp_files}

c++/src/capnp/any.h

@@ -221,9 +221,14 @@ struct AnyPointer {
     inline void setAs(std::initializer_list<ReaderFor<ListElementType<T>>> list);
     // Valid for T = List<?>.
 
+    template <typename T>
+    inline void setCanonicalAs(ReaderFor<T> value);
+
     inline void set(Reader value) { builder.copyFrom(value.reader); }
     // Set to a copy of another AnyPointer.
 
+    inline void setCanonical(Reader value) { builder.copyFrom(value.reader, true); }
+
     template <typename T>
     inline void adopt(Orphan<T>&& orphan);
     // Valid for T = any generated struct type, List<U>, Text, Data, DynamicList, DynamicStruct,
@@ -463,6 +468,10 @@ public:
     return List<AnyPointer>::Reader(_reader.getPointerSectionAsList());
   }
 
+  kj::Array<word> canonicalize() {
+    return _reader.canonicalize();
+  }
+
   Equality equals(AnyStruct::Reader right);
   bool operator==(AnyStruct::Reader right);
   inline bool operator!=(AnyStruct::Reader right) {
@@ -791,6 +800,11 @@ inline void AnyPointer::Builder::setAs(ReaderFor<T> value) {
   return _::PointerHelpers<T>::set(builder, value);
 }
 
+template <typename T>
+inline void AnyPointer::Builder::setCanonicalAs(ReaderFor<T> value) {
+  return _::PointerHelpers<T>::setCanonical(builder, value);
+}
+
 template <typename T>
 inline void AnyPointer::Builder::setAs(
     std::initializer_list<ReaderFor<ListElementType<T>>> list) {

c++/src/capnp/canonicalize-test.c++

+// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "message.h"
+#include "any.h"
+#include <kj/debug.h>
+#include <kj/test.h>
+#include "test-util.h"
+
+namespace capnp {
+namespace _ {  // private
+using test::TestLists;
+namespace {
+
+
+KJ_TEST("canonicalize yields cannonical message") {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<TestAllTypes>();
+
+  initTestMessage(root);
+
+  canonicalize(root.asReader());
+  //Will assert if canonicalize failed to do so
+}
+
+KJ_TEST("isCanonical requires pointer preorder") {
+  AlignedData<5> misorderedSegment = {{
+    //Struct pointer, data immediately follows, two pointer fields, no data
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
+    //Pointer field 1, pointing to the last entry, data size 1, no pointer
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Pointer field 2, pointing to the next entry, data size 2, no pointer
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Data for field 2
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Data for field 1
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(misorderedSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader outOfOrder(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!outOfOrder.isCanonical());
+}
+
+KJ_TEST("isCanonical requires dense packing") {
+   AlignedData<3> gapSegment = {{
+    //Struct pointer, data after a gap
+    0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //The gap
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data for field 1
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(gapSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader gap(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!gap.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects multi-segment messages") {
+  AlignedData<1> farPtr = {{
+    //Far pointer to next segment
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+  }};
+
+  AlignedData<2> farTarget = {{
+    //Struct pointer (needed to make the far pointer legal)
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Dummy data
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+
+  kj::ArrayPtr<const word> segments[2] = {
+    kj::arrayPtr(farPtr.words, 1),
+    kj::arrayPtr(farTarget.words, 2)
+  };
+
+  SegmentArrayMessageReader multiSegmentMessage(kj::arrayPtr(segments, 2));
+  KJ_ASSERT(!multiSegmentMessage.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects zero segment messages") {
+  SegmentArrayMessageReader zero(kj::arrayPtr((kj::ArrayPtr<const word>*)NULL,
+                                               0));
+  KJ_ASSERT(!zero.isCanonical());
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields") {
+  AlignedData<2> nonTruncatedSegment = {{
+    //Struct pointer, data immediately follows
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Default data value, should have been truncated
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedSegment.words, 3)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+
+KJ_TEST("upgraded lists can be canonicalized") {
+  AlignedData<7> upgradedList = {{
+    //Struct pointer, data immediately follows, 4 pointer fields, no data
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00,
+    //Three words of default pointers to get to the int16 list
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //List pointer, 3 int16s.
+    0x01, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00,
+    //First two elements
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x05, 0x06,
+    //Last element
+    0x07, 0x08, 0x09, 0x10, 0x00, 0x00, 0x00, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(upgradedList.words, 7)
+  };
+  SegmentArrayMessageReader upgraded(kj::arrayPtr(segments, 1));
+
+  auto root = upgraded.getRoot<TestLists>();
+  canonicalize(root);
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields in all list members") {
+  AlignedData<6> nonTruncatedList = {{
+    //List pointer, composite,
+    0x01, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
+    //Struct tag word, 2 structs, 2 data words per struct
+    0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedList.words, 6)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+}  // namespace
+}  // namespace _ (private)
+}  // namespace capnp

c++/src/capnp/layout.c++

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -357,6 +357,7 @@ struct WireHelpers {
       word* ptr = segment->allocate(amount);
 
       if (ptr == nullptr) {
+
         // Need to allocate in a new segment.  We'll need to allocate an extra pointer worth of
         // space to act as the landing pad for a far pointer.
 
@@ -1545,23 +1546,75 @@ struct WireHelpers {
 
   static SegmentAnd<word*> setStructPointer(
       SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, StructReader value,
-      BuilderArena* orphanArena = nullptr) {
-    WordCount dataSize = roundBitsUpToWords(value.dataSize);
-    WordCount totalSize = dataSize + value.pointerCount * WORDS_PER_POINTER;
+      BuilderArena* orphanArena = nullptr, bool canonical = false) {
+    ByteCount dataSize = roundBitsUpToBytes(value.dataSize);
+    WirePointerCount ptrCount = value.pointerCount;
+
+    if (canonical) {
+      // StructReaders should not have bitwidths other than 1, but let's be safe
+      KJ_REQUIRE((value.dataSize == 1 * BITS)
+                 || (value.dataSize % BITS_PER_BYTE == 0 * BITS));
+
+      // Handle the truncation case where it's a false in a 1-bit struct
+      if (value.dataSize == 1 * BITS) {
+        if (!value.getDataField<bool>(0 * ELEMENTS)) {
+          dataSize = 0 * BYTES;
+        }
+      } else {
+        // Truncate the data section
+        while (dataSize != 0 * BYTES) {
+          size_t end = (dataSize - 1 * BYTES) / BYTES;
+          ByteCount window = dataSize % BYTES_PER_WORD;
+          if (window == 0 * BYTES) {
+            window = BYTES_PER_WORD * WORDS;
+          }
+          size_t start = end + 1 - window / BYTES;
+          kj::ArrayPtr<const byte> lastWord = value.getDataSectionAsBlob().slice(start, end);
+          bool lastWordZero = true;
+          //TODO(MRM) once this is known to work, replace with fast memcmp
+          for (auto it = lastWord.begin(); it != lastWord.end(); it++) {
+            lastWordZero &= (*it == 0);
+          }
+          if (!lastWordZero) {
+            break;
+          } else {
+            dataSize -= window;
+          }
+        }
+      }
+      // Truncate pointer section
+      while ((ptrCount != 0 * POINTERS) &&
+             value.getPointerField(ptrCount - 1 * POINTERS).isNull()) {
+        ptrCount -= 1 * POINTERS;
+      }
+    }
+
+    WordCount dataWords = roundBytesUpToWords(dataSize);
+
+    WordCount totalSize = dataWords + ptrCount * WORDS_PER_POINTER;
 
     word* ptr = allocate(ref, segment, capTable, totalSize, WirePointer::STRUCT, orphanArena);
-    ref->structRef.set(dataSize, value.pointerCount);
+    ref->structRef.set(dataWords, ptrCount);
 
     if (value.dataSize == 1 * BITS) {
-      *reinterpret_cast<char*>(ptr) = value.getDataField<bool>(0 * ELEMENTS);
+      // Data size could be made 0 by truncation
+      if (dataSize != 0 * BYTES) {
+        *reinterpret_cast<char*>(ptr) = value.getDataField<bool>(0 * ELEMENTS);
+      }
     } else {
-      memcpy(ptr, value.data, value.dataSize / BITS_PER_BYTE / BYTES);
+      memcpy(ptr, value.data, dataSize / BYTES);
+      if (dataSize % BYTES_PER_WORD != 0 * BYTES) {
+        //Zero-pad the data if it didn't use the entire last word
+        byte* padStart = reinterpret_cast<byte*>(ptr) + (dataSize / BYTES);
+        bzero(padStart, (BYTES_PER_WORD * WORDS - (dataSize % BYTES_PER_WORD)) / BYTES);
+      }
     }
 
-    WirePointer* pointerSection = reinterpret_cast<WirePointer*>(ptr + dataSize);
-    for (uint i = 0; i < value.pointerCount / POINTERS; i++) {
+    WirePointer* pointerSection = reinterpret_cast<WirePointer*>(ptr + dataWords);
+    for (uint i = 0; i < ptrCount / POINTERS; i++) {
       copyPointer(segment, capTable, pointerSection + i,
-                  value.segment, value.capTable, value.pointers + i, value.nestingLimit);
+                  value.segment, value.capTable, value.pointers + i,
+                  value.nestingLimit, nullptr, canonical);
     }
 
     return { segment, ptr };
@@ -1584,7 +1637,7 @@ struct WireHelpers {
 
   static SegmentAnd<word*> setListPointer(
       SegmentBuilder* segment, CapTableBuilder* capTable, WirePointer* ref, ListReader value,
-      BuilderArena* orphanArena = nullptr) {
+      BuilderArena* orphanArena = nullptr, bool canonical = false) {
     WordCount totalSize = roundBitsUpToWords(value.elementCount * value.step);
 
     if (value.elementSize != ElementSize::INLINE_COMPOSITE) {
@@ -1598,7 +1651,7 @@ struct WireHelpers {
           copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(ptr) + i,
                       value.segment, value.capTable,
                       reinterpret_cast<const WirePointer*>(value.ptr) + i,
-                      value.nestingLimit);
+                      value.nestingLimit, nullptr, canonical);
         }
       } else {
         // List of data.
@@ -1610,31 +1663,74 @@ struct WireHelpers {
     } else {
       // List of structs.
       KJ_DASSERT(value.structDataSize % BITS_PER_WORD == 0 * BITS);
+
+      WordCount declDataSize = value.structDataSize / BITS_PER_WORD;
+      WirePointerCount declPointerCount = value.structPointerCount;
+
+      WordCount dataSize = 0 * WORDS;
+      WirePointerCount ptrCount = 0 * POINTERS;
+
+      if (canonical) {
+        for (auto ec = ElementCount(0); ec < value.elementCount; ec += 1 * ELEMENTS) {
+          auto se = value.getStructElement(ec);
+          WordCount localDataSize = declDataSize;
+          while (localDataSize != 0 * WORDS) {
+            size_t end = (localDataSize * BYTES_PER_WORD - 1 * BYTES) / BYTES;
+            ByteCount window = BYTES_PER_WORD * WORDS;
+            size_t start = end + 1 - window / BYTES;
+            kj::ArrayPtr<const byte> lastWord = se.getDataSectionAsBlob().slice(start, end);
+            bool lastWordZero = true;
+            //TODO(MRM) once this is known to work, replace with fast memcmp
+            for (auto it = lastWord.begin(); it != lastWord.end(); it++) {
+              lastWordZero &= (*it == 0);
+            }
+            if (!lastWordZero) {
+              break;
+            } else {
+              localDataSize -= WORDS;
+            }
+          }
+          if (localDataSize > dataSize) {
+            dataSize = localDataSize;
+          }
+          WirePointerCount localPtrCount = declPointerCount;
+          while ((localPtrCount != 0 * POINTERS) &&
+                 se.getPointerField(localPtrCount - 1 * POINTERS).isNull()) {
+            localPtrCount -= 1 * POINTERS;
+          }
+          if (localPtrCount > ptrCount) {
+            ptrCount = localPtrCount;
+          }
+        }
+        totalSize = (dataSize + ptrCount * WORDS_PER_POINTER) / ELEMENTS * value.elementCount;
+      } else {
+        dataSize = declDataSize;
+        ptrCount = declPointerCount;
+      }
+
       word* ptr = allocate(ref, segment, capTable, totalSize + POINTER_SIZE_IN_WORDS,
                            WirePointer::LIST, orphanArena);
       ref->listRef.setInlineComposite(totalSize);
 
-      WordCount dataSize = roundBitsUpToWords(value.structDataSize);
-      WirePointerCount pointerCount = value.structPointerCount;
-
       WirePointer* tag = reinterpret_cast<WirePointer*>(ptr);
       tag->setKindAndInlineCompositeListElementCount(WirePointer::STRUCT, value.elementCount);
-      tag->structRef.set(dataSize, pointerCount);
+      tag->structRef.set(dataSize, ptrCount);
       word* dst = ptr + POINTER_SIZE_IN_WORDS;
 
       const word* src = reinterpret_cast<const word*>(value.ptr);
       for (uint i = 0; i < value.elementCount / ELEMENTS; i++) {
-        memcpy(dst, src, value.structDataSize / BITS_PER_BYTE / BYTES);
+        memcpy(dst, src, dataSize * BYTES_PER_WORD / BYTES);
         dst += dataSize;
-        src += dataSize;
+        src += declDataSize;
 
-        for (uint j = 0; j < pointerCount / POINTERS; j++) {
+        for (uint j = 0; j < ptrCount / POINTERS; j++) {
           copyPointer(segment, capTable, reinterpret_cast<WirePointer*>(dst),
               value.segment, value.capTable, reinterpret_cast<const WirePointer*>(src),
-              value.nestingLimit);
+              value.nestingLimit, nullptr, canonical);
           dst += POINTER_SIZE_IN_WORDS;
           src += POINTER_SIZE_IN_WORDS;
         }
+        src += ((declPointerCount - ptrCount) * WORDS_PER_POINTER) / WORDS;
       }
 
       return { segment, ptr };
@@ -1644,16 +1740,18 @@ struct WireHelpers {
   static KJ_ALWAYS_INLINE(SegmentAnd<word*> copyPointer(
       SegmentBuilder* dstSegment, CapTableBuilder* dstCapTable, WirePointer* dst,
       SegmentReader* srcSegment, CapTableReader* srcCapTable, const WirePointer* src,
-      int nestingLimit, BuilderArena* orphanArena = nullptr)) {
+      int nestingLimit, BuilderArena* orphanArena = nullptr,
+      bool canonical = false)) {
     return copyPointer(dstSegment, dstCapTable, dst,
                        srcSegment, srcCapTable, src, src->target(),
-                       nestingLimit, orphanArena);
+                       nestingLimit, orphanArena, canonical);
   }
 
   static SegmentAnd<word*> copyPointer(
       SegmentBuilder* dstSegment, CapTableBuilder* dstCapTable, WirePointer* dst,
       SegmentReader* srcSegment, CapTableReader* srcCapTable, const WirePointer* src,
-      const word* srcTarget, int nestingLimit, BuilderArena* orphanArena = nullptr) {
+      const word* srcTarget, int nestingLimit,
+      BuilderArena* orphanArena = nullptr, bool canonical = false) {
     // Deep-copy the object pointed to by src into dst.  It turns out we can't reuse
     // readStructPointer(), etc. because they do type checking whereas here we want to accept any
     // valid pointer.
@@ -1690,7 +1788,7 @@ struct WireHelpers {
                          src->structRef.dataSize.get() * BITS_PER_WORD,
                          src->structRef.ptrCount.get(),
                          nestingLimit - 1),
-            orphanArena);
+            orphanArena, canonical);
 
       case WirePointer::LIST: {
         ElementSize elementSize = src->listRef.elementSize();
@@ -1738,7 +1836,7 @@ struct WireHelpers {
                          tag->structRef.dataSize.get() * BITS_PER_WORD,
                          tag->structRef.ptrCount.get(), ElementSize::INLINE_COMPOSITE,
                          nestingLimit - 1),
-              orphanArena);
+              orphanArena, canonical);
         } else {
           BitCount dataSize = dataBitsPerElement(elementSize) * ELEMENTS;
           WirePointerCount pointerCount = pointersPerElement(elementSize) * ELEMENTS;
@@ -1763,7 +1861,7 @@ struct WireHelpers {
           return setListPointer(dstSegment, dstCapTable, dst,
               ListReader(srcSegment, srcCapTable, ptr, elementCount, step, dataSize, pointerCount,
                          elementSize, nestingLimit - 1),
-              orphanArena);
+              orphanArena, canonical);
         }
       }
 
@@ -1777,6 +1875,11 @@ struct WireHelpers {
           goto useDefault;
         }
 
+        if (canonical) {
+          KJ_FAIL_REQUIRE("Cannot create a canonical message with a capability") {
+            break;
+          }
+        }
 #if !CAPNP_LITE
         KJ_IF_MAYBE(cap, srcCapTable->extractCap(src->capRef.index.get())) {
           setCapabilityPointer(dstSegment, dstCapTable, dst, kj::mv(*cap));
@@ -2280,12 +2383,12 @@ Data::Builder PointerBuilder::getBlob<Data>(const void* defaultValue, ByteCount
   return WireHelpers::getWritableDataPointer(pointer, segment, capTable, defaultValue, defaultSize);
 }
 
-void PointerBuilder::setStruct(const StructReader& value) {
-  WireHelpers::setStructPointer(segment, capTable, pointer, value);
+void PointerBuilder::setStruct(const StructReader& value, bool canonical) {
+  WireHelpers::setStructPointer(segment, capTable, pointer, value, nullptr, canonical);
 }
 
-void PointerBuilder::setList(const ListReader& value) {
-  WireHelpers::setListPointer(segment, capTable, pointer, value);
+void PointerBuilder::setList(const ListReader& value, bool canonical) {
+  WireHelpers::setListPointer(segment, capTable, pointer, value, nullptr, canonical);
 }
 
 #if !CAPNP_LITE
@@ -2342,7 +2445,7 @@ void PointerBuilder::transferFrom(PointerBuilder other) {
   memset(other.pointer, 0, sizeof(*other.pointer));
 }
 
-void PointerBuilder::copyFrom(PointerReader other) {
+void PointerBuilder::copyFrom(PointerReader other, bool canonical) {
   if (other.pointer == nullptr) {
     if (!pointer->isNull()) {
       WireHelpers::zeroObject(segment, capTable, pointer);
@@ -2350,7 +2453,9 @@ void PointerBuilder::copyFrom(PointerReader other) {
     }
   } else {
     WireHelpers::copyPointer(segment, capTable, pointer,
-                             other.segment, other.capTable, other.pointer, other.nestingLimit);
+                             other.segment, other.capTable, other.pointer, other.nestingLimit,
+                             nullptr,
+                             canonical);
   }
 }
 
@@ -2469,6 +2574,33 @@ PointerReader PointerReader::imbue(CapTableReader* capTable) const {
   return result;
 }
 
+bool PointerReader::isCanonical(const word **readHead) {
+  if (!this->pointer) {
+    // The pointer is null, so we are canonical and do not read
+    return true;
+  }
+
+  if (!this->pointer->isPositional()) {
+    // The pointer is a FAR or OTHER pointer, and is non-canonical
+    return false;
+  }
+
+  switch (this->getPointerType()) {
+    case PointerType::NULL_:
+      // The pointer is null, we are canonical and do not read
+      return true;
+    case PointerType::STRUCT:
+      bool dataTrunc, ptrTrunc;
+      return (this->getStruct(nullptr).isCanonical(readHead, readHead, &dataTrunc, &ptrTrunc)
+             && dataTrunc && ptrTrunc);
+    case PointerType::LIST:
+      return this->getListAnySize(nullptr).isCanonical(readHead);
+    case PointerType::CAPABILITY:
+      KJ_FAIL_ASSERT("Capabilities are not positional");
+  }
+  KJ_UNREACHABLE;
+}
+
 // =======================================================================================
 // StructBuilder
 
@@ -2600,6 +2732,19 @@ MessageSizeCounts StructReader::totalSize() const {
   return result;
 }
 
+kj::Array<word> StructReader::canonicalize() {
+  WordCount size = totalSize().wordCount + POINTER_SIZE_IN_WORDS;
+  kj::Array<word> backing = kj::heapArray<word>(size / WORDS);
+  memset(backing.begin(), 0, backing.asBytes().size());
+  FlatMessageBuilder builder(backing);
+  _::PointerHelpers<AnyPointer>::getInternalBuilder(builder.initRoot<AnyPointer>()).setStruct(*this, true);
+  KJ_ASSERT(builder.isCanonical());
+  auto output = builder.getSegmentsForOutput()[0];
+  kj::Array<word> trunc = kj::heapArray<word>(output.size());
+  memcpy(trunc.begin(), output.begin(), output.asBytes().size());
+  return trunc;
+}
+
 CapTableReader* StructReader::getCapTable() {
   return capTable;
 }
@@ -2610,6 +2755,49 @@ StructReader StructReader::imbue(CapTableReader* capTable) const {
   return result;
 }
 
+bool StructReader::isCanonical(const word **readHead,
+                               const word **ptrHead,
+                               bool *dataTrunc,
+                               bool *ptrTrunc) {
+  if (this->getLocation() != *readHead) {
+    // Our target area is not at the readHead, preorder fails
+    return false;
+  }
+
+  if (this->getDataSectionSize() % BITS_PER_WORD != 0 * BITS) {
+    // Using legacy non-word-size structs, reject
+    return false;
+  }
+  WordCount32 dataSize = this->getDataSectionSize() / BITS_PER_WORD;
+
+  // Mark whether the struct is properly truncated
+  if (dataSize != 0 * WORDS) {
+    *dataTrunc = this->getDataField<uint64_t>((dataSize - 1 * WORDS) / WORDS * ELEMENTS) != 0;
+  } else {
+    *dataTrunc = true;
+  }
+
+  if (this->pointerCount != 0 * POINTERS) {
+    *ptrTrunc  = !this->getPointerField(this->pointerCount - 1 * POINTERS).isNull();
+  } else {
+    *ptrTrunc = true;
+  }
+
+  // Advance the read head
+  *readHead += (dataSize + (this->pointerCount * WORDS_PER_POINTER)) / WORDS;
+
+  // Check each pointer field for canonicity
+  for (WirePointerCount16 ptrIndex = 0 * POINTERS;
+       ptrIndex < this->pointerCount;
+       ptrIndex += POINTERS) {
+    if (!this->getPointerField(ptrIndex).isCanonical(ptrHead)) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 // =======================================================================================
 // ListBuilder
 
@@ -2749,6 +2937,74 @@ ListReader ListReader::imbue(CapTableReader* capTable) const {
   return result;
 }
 
+bool ListReader::isCanonical(const word **readHead) {
+  switch (this->getElementSize()) {
+    case ElementSize::INLINE_COMPOSITE: {
+      *readHead += 1;
+      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
+        // The next word to read is the tag word, but the pointer is in
+        // front of it, so our check is slightly different
+        return false;
+      }
+      if (this->structDataSize % BITS_PER_WORD != 0 * BITS) {
+        return false;
+      }
+      auto structSize = (this->structDataSize / BITS_PER_WORD) +
+                        (this->structPointerCount * WORDS_PER_POINTER);
+      auto listEnd = *readHead + (this->elementCount / ELEMENTS * structSize) / WORDS;
+      auto pointerHead = listEnd;
+      bool listDataTrunc = false;
+      bool listPtrTrunc = false;
+      for (ElementCount ec = ElementCount(0);
+           ec < this->elementCount;
+           ec += 1 * ELEMENTS) {
+        bool dataTrunc, ptrTrunc;
+        if (!this->getStructElement(ec).isCanonical(readHead,
+                                                    &pointerHead,
+                                                    &dataTrunc,
+                                                    &ptrTrunc)) {
+          return false;
+        }
+        listDataTrunc |= dataTrunc;
+        listPtrTrunc  |= ptrTrunc;
+      }
+      KJ_REQUIRE(*readHead == listEnd, *readHead, listEnd);
+      *readHead = pointerHead;
+      return listDataTrunc && listPtrTrunc;
+    }
+    case ElementSize::POINTER: {
+      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
+        return false;
+      }
+      *readHead += this->elementCount * (POINTERS / ELEMENTS) * WORDS_PER_POINTER / WORDS;
+      for (ElementCount ec = ElementCount(0);
+           ec < this->elementCount;
+           ec += 1 * ELEMENTS) {
+        if (!this->getPointerElement(ec).isCanonical(readHead)) {
+          return false;
+        }
+      }
+      return true;
+    }
+    default: {
+      if (reinterpret_cast<const word*>(this->ptr) != *readHead) {
+        return false;
+      }
+
+      auto bitSize = this->elementCount *
+                     dataBitsPerElement(this->elementSize);
+      auto wordSize = bitSize / BITS_PER_WORD;
+      if (bitSize % BITS_PER_WORD != 0 * BITS) {
+        wordSize = wordSize + 1 * WORDS;
+      }
+
+      *readHead += wordSize;
+      return true;
+    }
+  }
+  KJ_UNREACHABLE;
+}
+
 // =======================================================================================
 // OrphanBuilder
 

c++/src/capnp/layout.h

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -339,8 +339,8 @@ public:
   // Init methods:  Initialize the pointer to a newly-allocated object, discarding the existing
   // object.
 
-  void setStruct(const StructReader& value);
-  void setList(const ListReader& value);
+  void setStruct(const StructReader& value, bool canonical = false);
+  void setList(const ListReader& value, bool canonical = false);
   template <typename T> void setBlob(typename T::Reader value);
 #if !CAPNP_LITE
   void setCapability(kj::Own<ClientHook>&& cap);
@@ -360,8 +360,10 @@ public:
   void transferFrom(PointerBuilder other);
   // Equivalent to `adopt(other.disown())`.
 
-  void copyFrom(PointerReader other);
+  void copyFrom(PointerReader other, bool canonical = false);
   // Equivalent to `set(other.get())`.
+  // If you set the canonical flag, it will attempt to lay the target out
+  // canonically, provided enough space is available.
 
   PointerReader asReader() const;
 
@@ -436,6 +438,13 @@ public:
   PointerReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;      // Memory segment in which the pointer resides.
   CapTableReader* capTable;    // Table of capability indexes.
@@ -562,6 +571,8 @@ public:
   inline kj::ArrayPtr<const byte> getDataSectionAsBlob();
   inline _::ListReader getPointerSectionAsList();
 
+  kj::Array<word> canonicalize();
+
   template <typename T>
   KJ_ALWAYS_INLINE(bool hasDataField(ElementCount offset) const);
   // Return true if the field is set to something other than its default value.
@@ -595,6 +606,21 @@ public:
   StructReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead, const word **ptrHead,
+                   bool *dataTrunc, bool *ptrTrunc);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+  //
+  // If this function returns false, the struct is non-canonical. If it
+  // returns true, then:
+  // * If it is a composite in a list, it is canonical if at least one struct
+  //   in the list outputs dataTrunc = 1, and at least one outputs ptrTrunc = 1
+  // * If it is derived from a struct pointer, it is canonical if
+  //   dataTrunc = 1 AND ptrTrunc = 1
+
 private:
   SegmentReader* segment;    // Memory segment in which the struct resides.
   CapTableReader* capTable;  // Table of capability indexes.
@@ -743,6 +769,13 @@ public:
   ListReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;    // Memory segment in which the list resides.
   CapTableReader* capTable;  // Table of capability indexes.

c++/src/capnp/message.c++

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -53,6 +53,34 @@ MessageReader::~MessageReader() noexcept(false) {
   }
 }
 
+bool MessageReader::isCanonical() {
+  if (!allocatedArena) {
+    static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
+        "arenaSpace is too small to hold a ReaderArena.  Please increase it.  This will break "
+        "ABI compatibility.");
+    new(arena()) _::ReaderArena(this);
+    allocatedArena = true;
+  }
+
+  _::SegmentReader *segment = arena()->tryGetSegment(_::SegmentId(0));
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(),
+                                   this->getOptions().nestingLimit)
+                                  .isCanonical(&readHead);
+}
+
+
 AnyPointer::Reader MessageReader::getRootInternal() {
   if (!allocatedArena) {
     static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
@@ -132,6 +160,24 @@ Orphanage MessageBuilder::getOrphanage() {
   return Orphanage(arena(), arena()->getLocalCapTable());
 }
 
+bool MessageBuilder::isCanonical() {
+  _::SegmentReader *segment = getRootSegment();
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(), kj::maxValue)
+                                  .isCanonical(&readHead);
+}
+
 // =======================================================================================
 
 SegmentArrayMessageReader::SegmentArrayMessageReader(

c++/src/capnp/message.h

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -22,6 +22,7 @@
 #include <kj/common.h>
 #include <kj/memory.h>
 #include <kj/mutex.h>
+#include <kj/debug.h>
 #include "common.h"
 #include "layout.h"
 #include "any.h"
@@ -118,6 +119,9 @@ public:
   // RootType in this case must be DynamicStruct, and you must #include <capnp/dynamic.h> to
   // use this.
 
+  bool isCanonical();
+  // Returns whether the message encoded in the reader is in canonical form.
+
 private:
   ReaderOptions options;
 
@@ -220,6 +224,9 @@ public:
 
   Orphanage getOrphanage();
 
+  bool isCanonical();
+  // Check whether the message builder is in canonical form
+
 private:
   void* arenaSpace[22];
   // Space in which we can construct a BuilderArena.  We don't use BuilderArena directly here
@@ -491,6 +498,11 @@ static typename Type::Reader defaultValue() {
   return typename Type::Reader(_::StructReader());
 }
 
+template <typename T>
+kj::Array<word> canonicalize(T&& reader) {
+    return _::PointerHelpers<FromReader<T>>::getInternalReader(reader).canonicalize();
+}
+
 }  // namespace capnp
 
 #endif  // CAPNP_MESSAGE_H_

c++/src/capnp/pointer-helpers.h

@@ -48,6 +48,9 @@ struct PointerHelpers<T, Kind::STRUCT> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setStruct(value._reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setStruct(value._reader, true);
+  }
   static inline typename T::Builder init(PointerBuilder builder) {
     return typename T::Builder(builder.initStruct(structSize<T>()));
   }
@@ -78,6 +81,9 @@ struct PointerHelpers<List<T>, Kind::LIST> {
   static inline void set(PointerBuilder builder, typename List<T>::Reader value) {
     builder.setList(value.reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename List<T>::Reader value) {
+    builder.setList(value.reader, true);
+  }
   static void set(PointerBuilder builder, kj::ArrayPtr<const ReaderFor<T>> value) {
     auto l = init(builder, value.size());
     uint i = 0;
@@ -117,6 +123,9 @@ struct PointerHelpers<T, Kind::BLOB> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setBlob<T>(value);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setBlob<T>(value);
+  }
   static inline typename T::Builder init(PointerBuilder builder, uint size) {
     return builder.initBlob<T>(size * BYTES);
   }