Add Canonicalization

The user facing API is in MessageReader and MessageBuilder

{MessageBuilder,MessageReader}::isCanonical verifies the canonicity of a
message. This is both useful for debugging and for knowing if a received
message can be used for hashes, bytewise equality, etc.

MessageBuilder::canonicalRoot(Reader) can be used to write a canonical
message on a best effort basis, and checks itself using isCanonical.
It should succeed as long as the MessageBuilder in question:
* Has a first segment which is long enough to contain the message
* Has not been used before

Tests have been added in canonicalize-test.c++ which verify that for
crafted examples of canonicalization errors, isCanonical will reject,
and for a canonicalized version of the standard test message, it will
accept.

c++/src/capnp/CMakeLists.txt

@@ -200,6 +200,7 @@ if(BUILD_TESTING)
     orphan-test.c++
     serialize-test.c++
     serialize-packed-test.c++
+    canonicalize-test.c++
     fuzz-test.c++
     test-util.c++
     ${test_capnp_cpp_files}

c++/src/capnp/any.h

@@ -221,9 +221,14 @@ struct AnyPointer {
     inline void setAs(std::initializer_list<ReaderFor<ListElementType<T>>> list);
     // Valid for T = List<?>.
 
+    template <typename T>
+    inline void setCanonicalAs(ReaderFor<T> value);
+
     inline void set(Reader value) { builder.copyFrom(value.reader); }
     // Set to a copy of another AnyPointer.
 
+    inline void setCanonical(Reader value) { builder.copyFrom(value.reader, true); }
+
     template <typename T>
     inline void adopt(Orphan<T>&& orphan);
     // Valid for T = any generated struct type, List<U>, Text, Data, DynamicList, DynamicStruct,
@@ -793,6 +798,11 @@ inline void AnyPointer::Builder::setAs(ReaderFor<T> value) {
   return _::PointerHelpers<T>::set(builder, value);
 }
 
+template <typename T>
+inline void AnyPointer::Builder::setCanonicalAs(ReaderFor<T> value) {
+  return _::PointerHelpers<T>::setCanonical(builder, value);
+}
+
 template <typename T>
 inline void AnyPointer::Builder::setAs(
     std::initializer_list<ReaderFor<ListElementType<T>>> list) {

c++/src/capnp/canonicalize-test.c++

+// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "message.h"
+#include "any.h"
+#include <kj/debug.h>
+#include <kj/test.h>
+#include "test-util.h"
+
+namespace capnp {
+namespace _ {  // private
+namespace {
+
+KJ_TEST("canonicalize yields cannonical message") {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<TestAllTypes>();
+
+  initTestMessage(root);
+
+  MallocMessageBuilder canonicalMessage;
+  canonicalMessage.canonicalRoot(builder.getRoot<AnyPointer>().asReader());
+  KJ_ASSERT(canonicalMessage.isCanonical());
+}
+
+KJ_TEST("isCanonical requires pointer preorder") {
+  AlignedData<5> misorderedSegment = {{
+    //Struct pointer, data immediately follows, two pointer fields, no data
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
+    //Pointer field 1, pointing to the last entry, data size 1, no pointer
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Pointer field 2, pointing to the next entry, data size 2, no pointer
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Data for field 2
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Data for field 1
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(misorderedSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader outOfOrder(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!outOfOrder.isCanonical());
+}
+
+KJ_TEST("isCanonical requires dense packing") {
+   AlignedData<3> gapSegment = {{
+    //Struct pointer, data after a gap
+    0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //The gap
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data for field 1
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(gapSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader gap(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!gap.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects multi-segment messages") {
+  AlignedData<1> farPtr = {{
+    //Far pointer to next segment
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+  }};
+
+  AlignedData<2> farTarget = {{
+    //Struct pointer (needed to make the far pointer legal)
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Dummy data
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+
+  kj::ArrayPtr<const word> segments[2] = {
+    kj::arrayPtr(farPtr.words, 1),
+    kj::arrayPtr(farTarget.words, 2)
+  };
+
+  SegmentArrayMessageReader multiSegmentMessage(kj::arrayPtr(segments, 2));
+  KJ_ASSERT(!multiSegmentMessage.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects zero segment messages") {
+  SegmentArrayMessageReader zero(kj::arrayPtr((kj::ArrayPtr<const word>*)NULL,
+                                               0));
+  KJ_ASSERT(!zero.isCanonical());
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields") {
+  AlignedData<2> nonTruncatedSegment = {{
+    //Struct pointer, data immediately follows
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Default data value, should have been truncated
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedSegment.words, 3)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields\
+          in all list members") {
+  AlignedData<6> nonTruncatedList = {{
+    //List pointer, composite,
+    0x01, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
+    //Struct tag word, 2 structs, 2 data words per struct
+    0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedList.words, 6)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+}  // namespace
+}  // namespace _ (private)
+}  // namespace capnp

c++/src/capnp/layout.h

@@ -339,9 +339,9 @@ public:
   // Init methods:  Initialize the pointer to a newly-allocated object, discarding the existing
   // object.
 
-  void setStruct(const StructReader& value);
-  void setList(const ListReader& value);
-  template <typename T> void setBlob(typename T::Reader value);
+  void setStruct(const StructReader& value, bool canonical = false);
+  void setList(const ListReader& value, bool canonical = false);
+  template <typename T> void setBlob(typename T::Reader value, bool canonical = false);
 #if !CAPNP_LITE
   void setCapability(kj::Own<ClientHook>&& cap);
 #endif  // !CAPNP_LITE
@@ -360,8 +360,10 @@ public:
   void transferFrom(PointerBuilder other);
   // Equivalent to `adopt(other.disown())`.
 
-  void copyFrom(PointerReader other);
+  void copyFrom(PointerReader other, bool canonical = false);
   // Equivalent to `set(other.get())`.
+  // If you set the canonical flag, it will attempt to lay the target out
+  // canonically, provided enough space is available.
 
   PointerReader asReader() const;
 
@@ -436,6 +438,13 @@ public:
   PointerReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;      // Memory segment in which the pointer resides.
   CapTableReader* capTable;    // Table of capability indexes.
@@ -595,6 +604,21 @@ public:
   StructReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead, const word **ptrHead,
+                   bool *dataTrunc, bool *ptrTrunc);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+  //
+  // If this function returns false, the struct is non-canonical. If it
+  // returns true, then:
+  // * If it is a composite in a list, it is canonical if at least one struct
+  //   in the list outputs dataTrunc = 1, and at least one outputs ptrTrunc = 1
+  // * If it is derived from a struct pointer, it is canonical if
+  //   dataTrunc = 1 AND ptrTrunc = 1
+
 private:
   SegmentReader* segment;    // Memory segment in which the struct resides.
   CapTableReader* capTable;  // Table of capability indexes.
@@ -758,6 +782,13 @@ public:
   ListReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;    // Memory segment in which the list resides.
   CapTableReader* capTable;  // Table of capability indexes.
@@ -915,12 +946,12 @@ private:
 
 // These are defined in the source file.
 template <> typename Text::Builder PointerBuilder::initBlob<Text>(ByteCount size);
-template <> void PointerBuilder::setBlob<Text>(typename Text::Reader value);
+template <> void PointerBuilder::setBlob<Text>(typename Text::Reader value, bool canonical);
 template <> typename Text::Builder PointerBuilder::getBlob<Text>(const void* defaultValue, ByteCount defaultSize);
 template <> typename Text::Reader PointerReader::getBlob<Text>(const void* defaultValue, ByteCount defaultSize) const;
 
 template <> typename Data::Builder PointerBuilder::initBlob<Data>(ByteCount size);
-template <> void PointerBuilder::setBlob<Data>(typename Data::Reader value);
+template <> void PointerBuilder::setBlob<Data>(typename Data::Reader value, bool canonical);
 template <> typename Data::Builder PointerBuilder::getBlob<Data>(const void* defaultValue, ByteCount defaultSize);
 template <> typename Data::Reader PointerReader::getBlob<Data>(const void* defaultValue, ByteCount defaultSize) const;
 

c++/src/capnp/message.c++

@@ -29,6 +29,7 @@
 #include <string>
 #include <vector>
 #include <errno.h>
+#include <climits>
 
 namespace capnp {
 
@@ -51,6 +52,34 @@ MessageReader::~MessageReader() noexcept(false) {
   }
 }
 
+bool MessageReader::isCanonical() {
+  if (!allocatedArena) {
+    static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
+        "arenaSpace is too small to hold a ReaderArena.  Please increase it.  This will break "
+        "ABI compatibility.");
+    new(arena()) _::ReaderArena(this);
+    allocatedArena = true;
+  }
+
+  _::SegmentReader *segment = arena()->tryGetSegment(_::SegmentId(0));
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(),
+                                   this->getOptions().nestingLimit)
+                                  .isCanonical(&readHead);
+}
+
+
 AnyPointer::Reader MessageReader::getRootInternal() {
   if (!allocatedArena) {
     static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
@@ -130,6 +159,25 @@ Orphanage MessageBuilder::getOrphanage() {
   return Orphanage(arena(), arena()->getLocalCapTable());
 }
 
+bool MessageBuilder::isCanonical() {
+  _::SegmentReader *segment = getRootSegment();
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(),
+                                   INT_MAX)
+                                  .isCanonical(&readHead);
+}
+
 // =======================================================================================
 
 SegmentArrayMessageReader::SegmentArrayMessageReader(

c++/src/capnp/message.h

@@ -22,6 +22,7 @@
 #include <kj/common.h>
 #include <kj/memory.h>
 #include <kj/mutex.h>
+#include <kj/debug.h>
 #include "common.h"
 #include "layout.h"
 #include "any.h"
@@ -118,6 +119,9 @@ public:
   // RootType in this case must be DynamicStruct, and you must #include <capnp/dynamic.h> to
   // use this.
 
+  bool isCanonical();
+  // Returns whether the message encoded in the reader is in canonical form.
+
 private:
   ReaderOptions options;
 
@@ -195,6 +199,12 @@ public:
   void setRoot(Reader&& value);
   // Set the root struct to a deep copy of the given struct.
 
+  template <typename Reader>
+  void canonicalRoot(Reader&& value);
+  // Set the root to a canonical deep copy of the struct.
+  // will likely only work if the builder has not yet been used, and has
+  // been set up with an arena with a segment as big as value.targetSize();
+
   template <typename RootType>
   typename RootType::Builder getRoot();
   // Get the root struct of the message, interpreting it as the given struct type.
@@ -220,6 +230,9 @@ public:
 
   Orphanage getOrphanage();
 
+  bool isCanonical();
+  // Check whether the message builder is in canonical form
+
 private:
   void* arenaSpace[22];
   // Space in which we can construct a BuilderArena.  We don't use BuilderArena directly here
@@ -461,6 +474,13 @@ typename RootType::Builder MessageBuilder::initRoot(SchemaType schema) {
   return getRootInternal().initAs<RootType>(schema);
 }
 
+template <typename Reader>
+void MessageBuilder::canonicalRoot(Reader&& value) {
+  auto target = initRoot<AnyPointer>();
+  target.setCanonical(value);
+  KJ_ASSERT(isCanonical());
+}
+
 template <typename RootType>
 typename RootType::Reader readMessageUnchecked(const word* data) {
   return AnyPointer::Reader(_::PointerReader::getRootUnchecked(data)).getAs<RootType>();

c++/src/capnp/pointer-helpers.h

@@ -48,6 +48,9 @@ struct PointerHelpers<T, Kind::STRUCT> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setStruct(value._reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setStruct(value._reader, true);
+  }
   static inline typename T::Builder init(PointerBuilder builder) {
     return typename T::Builder(builder.initStruct(structSize<T>()));
   }
@@ -78,6 +81,9 @@ struct PointerHelpers<List<T>, Kind::LIST> {
   static inline void set(PointerBuilder builder, typename List<T>::Reader value) {
     builder.setList(value.reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename List<T>::Reader value) {
+    builder.setList(value.reader, true);
+  }
   static void set(PointerBuilder builder, kj::ArrayPtr<const ReaderFor<T>> value) {
     auto l = init(builder, value.size());
     uint i = 0;
@@ -117,6 +123,9 @@ struct PointerHelpers<T, Kind::BLOB> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setBlob<T>(value);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setBlob<T>(value);
+  }
   static inline typename T::Builder init(PointerBuilder builder, uint size) {
     return builder.initBlob<T>(size * BYTES);
   }