Merge pull request #293 from maurer/canonicalize-pr

Add Canonicalization

CONTRIBUTORS

@@ -15,6 +15,7 @@ Kamal Marhubi <kamal@marhubi.com>: JSON parser
 Oliver Kuckertz <oliver.kuckertz@mologie.de>: FdObserver POLLPRI support
 Harris Hancock <vortrab@gmail.com>: MSVC support
 Branislav Katreniak <branislav.katreniak@digitalstrom.com>: JSON decode
+Matthew Maurer <matthew.r.maurer@gmail.com>: Canonicalization Support
 
 This file does not list people who maintain their own Cap'n Proto
 implementations as separate projects.  Those people are awesome too!  :)

c++/src/capnp/CMakeLists.txt

@@ -210,6 +210,7 @@ if(BUILD_TESTING)
     orphan-test.c++
     serialize-test.c++
     serialize-packed-test.c++
+    canonicalize-test.c++
     fuzz-test.c++
     test-util.c++
     ${test_capnp_cpp_files}

c++/src/capnp/any.h

@@ -221,9 +221,14 @@ struct AnyPointer {
     inline void setAs(std::initializer_list<ReaderFor<ListElementType<T>>> list);
     // Valid for T = List<?>.
 
+    template <typename T>
+    inline void setCanonicalAs(ReaderFor<T> value);
+
     inline void set(Reader value) { builder.copyFrom(value.reader); }
     // Set to a copy of another AnyPointer.
 
+    inline void setCanonical(Reader value) { builder.copyFrom(value.reader, true); }
+
     template <typename T>
     inline void adopt(Orphan<T>&& orphan);
     // Valid for T = any generated struct type, List<U>, Text, Data, DynamicList, DynamicStruct,
@@ -463,6 +468,10 @@ public:
     return List<AnyPointer>::Reader(_reader.getPointerSectionAsList());
   }
 
+  kj::Array<word> canonicalize() {
+    return _reader.canonicalize();
+  }
+
   Equality equals(AnyStruct::Reader right);
   bool operator==(AnyStruct::Reader right);
   inline bool operator!=(AnyStruct::Reader right) {
@@ -791,6 +800,11 @@ inline void AnyPointer::Builder::setAs(ReaderFor<T> value) {
   return _::PointerHelpers<T>::set(builder, value);
 }
 
+template <typename T>
+inline void AnyPointer::Builder::setCanonicalAs(ReaderFor<T> value) {
+  return _::PointerHelpers<T>::setCanonical(builder, value);
+}
+
 template <typename T>
 inline void AnyPointer::Builder::setAs(
     std::initializer_list<ReaderFor<ListElementType<T>>> list) {

c++/src/capnp/canonicalize-test.c++

+// Copyright (c) 2016 Sandstorm Development Group, Inc. and contributors
+// Licensed under the MIT License:
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+#include "message.h"
+#include "any.h"
+#include <kj/debug.h>
+#include <kj/test.h>
+#include "test-util.h"
+
+namespace capnp {
+namespace _ {  // private
+using test::TestLists;
+namespace {
+
+
+KJ_TEST("canonicalize yields cannonical message") {
+  MallocMessageBuilder builder;
+  auto root = builder.initRoot<TestAllTypes>();
+
+  initTestMessage(root);
+
+  canonicalize(root.asReader());
+  //Will assert if canonicalize failed to do so
+}
+
+KJ_TEST("isCanonical requires pointer preorder") {
+  AlignedData<5> misorderedSegment = {{
+    //Struct pointer, data immediately follows, two pointer fields, no data
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00,
+    //Pointer field 1, pointing to the last entry, data size 1, no pointer
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Pointer field 2, pointing to the next entry, data size 2, no pointer
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Data for field 2
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Data for field 1
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(misorderedSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader outOfOrder(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!outOfOrder.isCanonical());
+}
+
+KJ_TEST("isCanonical requires dense packing") {
+   AlignedData<3> gapSegment = {{
+    //Struct pointer, data after a gap
+    0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //The gap
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data for field 1
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {kj::arrayPtr(gapSegment.words,
+                                                       3)};
+  SegmentArrayMessageReader gap(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!gap.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects multi-segment messages") {
+  AlignedData<1> farPtr = {{
+    //Far pointer to next segment
+    0x02, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+  }};
+
+  AlignedData<2> farTarget = {{
+    //Struct pointer (needed to make the far pointer legal)
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Dummy data
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+  }};
+
+  kj::ArrayPtr<const word> segments[2] = {
+    kj::arrayPtr(farPtr.words, 1),
+    kj::arrayPtr(farTarget.words, 2)
+  };
+
+  SegmentArrayMessageReader multiSegmentMessage(kj::arrayPtr(segments, 2));
+  KJ_ASSERT(!multiSegmentMessage.isCanonical());
+}
+
+KJ_TEST("isCanonical rejects zero segment messages") {
+  SegmentArrayMessageReader zero(kj::arrayPtr((kj::ArrayPtr<const word>*)NULL,
+                                               0));
+  KJ_ASSERT(!zero.isCanonical());
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields") {
+  AlignedData<2> nonTruncatedSegment = {{
+    //Struct pointer, data immediately follows
+    0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
+    //Default data value, should have been truncated
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedSegment.words, 3)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+
+KJ_TEST("upgraded lists can be canonicalized") {
+  AlignedData<7> upgradedList = {{
+    //Struct pointer, data immediately follows, 4 pointer fields, no data
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00,
+    //Three words of default pointers to get to the int16 list
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //List pointer, 3 int16s.
+    0x01, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00, 0x00,
+    //First two elements
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x05, 0x06,
+    //Last element
+    0x07, 0x08, 0x09, 0x10, 0x00, 0x00, 0x00, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(upgradedList.words, 7)
+  };
+  SegmentArrayMessageReader upgraded(kj::arrayPtr(segments, 1));
+
+  auto root = upgraded.getRoot<TestLists>();
+  canonicalize(root);
+}
+
+KJ_TEST("isCanonical requires truncation of 0-valued struct fields in all list members") {
+  AlignedData<6> nonTruncatedList = {{
+    //List pointer, composite,
+    0x01, 0x00, 0x00, 0x00, 0x27, 0x00, 0x00, 0x00,
+    //Struct tag word, 2 structs, 2 data words per struct
+    0x08, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    //Data word non-null
+    0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
+    //Null trailing word
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
+  }};
+  kj::ArrayPtr<const word> segments[1] = {
+    kj::arrayPtr(nonTruncatedList.words, 6)
+  };
+  SegmentArrayMessageReader nonTruncated(kj::arrayPtr(segments, 1));
+
+  KJ_ASSERT(!nonTruncated.isCanonical());
+}
+}  // namespace
+}  // namespace _ (private)
+}  // namespace capnp

c++/src/capnp/layout.h

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -339,8 +339,8 @@ public:
   // Init methods:  Initialize the pointer to a newly-allocated object, discarding the existing
   // object.
 
-  void setStruct(const StructReader& value);
-  void setList(const ListReader& value);
+  void setStruct(const StructReader& value, bool canonical = false);
+  void setList(const ListReader& value, bool canonical = false);
   template <typename T> void setBlob(typename T::Reader value);
 #if !CAPNP_LITE
   void setCapability(kj::Own<ClientHook>&& cap);
@@ -360,8 +360,10 @@ public:
   void transferFrom(PointerBuilder other);
   // Equivalent to `adopt(other.disown())`.
 
-  void copyFrom(PointerReader other);
+  void copyFrom(PointerReader other, bool canonical = false);
   // Equivalent to `set(other.get())`.
+  // If you set the canonical flag, it will attempt to lay the target out
+  // canonically, provided enough space is available.
 
   PointerReader asReader() const;
 
@@ -436,6 +438,13 @@ public:
   PointerReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;      // Memory segment in which the pointer resides.
   CapTableReader* capTable;    // Table of capability indexes.
@@ -562,6 +571,8 @@ public:
   inline kj::ArrayPtr<const byte> getDataSectionAsBlob();
   inline _::ListReader getPointerSectionAsList();
 
+  kj::Array<word> canonicalize();
+
   template <typename T>
   KJ_ALWAYS_INLINE(bool hasDataField(ElementCount offset) const);
   // Return true if the field is set to something other than its default value.
@@ -595,6 +606,21 @@ public:
   StructReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead, const word **ptrHead,
+                   bool *dataTrunc, bool *ptrTrunc);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+  //
+  // If this function returns false, the struct is non-canonical. If it
+  // returns true, then:
+  // * If it is a composite in a list, it is canonical if at least one struct
+  //   in the list outputs dataTrunc = 1, and at least one outputs ptrTrunc = 1
+  // * If it is derived from a struct pointer, it is canonical if
+  //   dataTrunc = 1 AND ptrTrunc = 1
+
 private:
   SegmentReader* segment;    // Memory segment in which the struct resides.
   CapTableReader* capTable;  // Table of capability indexes.
@@ -743,6 +769,13 @@ public:
   ListReader imbue(CapTableReader* capTable) const;
   // Return a copy of this reader except using the given capability context.
 
+  bool isCanonical(const word **readHead);
+  // Validate this pointer's canonicity, subject to the conditions:
+  // * All data to the left of readHead has been read thus far (for pointer
+  //   ordering)
+  // * All pointers in preorder have already been checked
+  // * This pointer is in the first and only segment of the message
+
 private:
   SegmentReader* segment;    // Memory segment in which the list resides.
   CapTableReader* capTable;  // Table of capability indexes.

c++/src/capnp/message.c++

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -53,6 +53,34 @@ MessageReader::~MessageReader() noexcept(false) {
   }
 }
 
+bool MessageReader::isCanonical() {
+  if (!allocatedArena) {
+    static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
+        "arenaSpace is too small to hold a ReaderArena.  Please increase it.  This will break "
+        "ABI compatibility.");
+    new(arena()) _::ReaderArena(this);
+    allocatedArena = true;
+  }
+
+  _::SegmentReader *segment = arena()->tryGetSegment(_::SegmentId(0));
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(),
+                                   this->getOptions().nestingLimit)
+                                  .isCanonical(&readHead);
+}
+
+
 AnyPointer::Reader MessageReader::getRootInternal() {
   if (!allocatedArena) {
     static_assert(sizeof(_::ReaderArena) <= sizeof(arenaSpace),
@@ -132,6 +160,24 @@ Orphanage MessageBuilder::getOrphanage() {
   return Orphanage(arena(), arena()->getLocalCapTable());
 }
 
+bool MessageBuilder::isCanonical() {
+  _::SegmentReader *segment = getRootSegment();
+
+  if (segment == NULL) {
+    // The message has no segments
+    return false;
+  }
+
+  if (arena()->tryGetSegment(_::SegmentId(1))) {
+    // The message has more than one segment
+    return false;
+  }
+
+  const word* readHead = segment->getStartPtr() + 1;
+  return _::PointerReader::getRoot(segment, nullptr, segment->getStartPtr(), kj::maxValue)
+                                  .isCanonical(&readHead);
+}
+
 // =======================================================================================
 
 SegmentArrayMessageReader::SegmentArrayMessageReader(

c++/src/capnp/message.h

-// Copyright (c) 2013-2014 Sandstorm Development Group, Inc. and contributors
+// Copyright (c) 2013-2016 Sandstorm Development Group, Inc. and contributors
 // Licensed under the MIT License:
 //
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -22,6 +22,7 @@
 #include <kj/common.h>
 #include <kj/memory.h>
 #include <kj/mutex.h>
+#include <kj/debug.h>
 #include "common.h"
 #include "layout.h"
 #include "any.h"
@@ -118,6 +119,9 @@ public:
   // RootType in this case must be DynamicStruct, and you must #include <capnp/dynamic.h> to
   // use this.
 
+  bool isCanonical();
+  // Returns whether the message encoded in the reader is in canonical form.
+
 private:
   ReaderOptions options;
 
@@ -220,6 +224,9 @@ public:
 
   Orphanage getOrphanage();
 
+  bool isCanonical();
+  // Check whether the message builder is in canonical form
+
 private:
   void* arenaSpace[22];
   // Space in which we can construct a BuilderArena.  We don't use BuilderArena directly here
@@ -491,6 +498,11 @@ static typename Type::Reader defaultValue() {
   return typename Type::Reader(_::StructReader());
 }
 
+template <typename T>
+kj::Array<word> canonicalize(T&& reader) {
+    return _::PointerHelpers<FromReader<T>>::getInternalReader(reader).canonicalize();
+}
+
 }  // namespace capnp
 
 #endif  // CAPNP_MESSAGE_H_

c++/src/capnp/pointer-helpers.h

@@ -48,6 +48,9 @@ struct PointerHelpers<T, Kind::STRUCT> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setStruct(value._reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setStruct(value._reader, true);
+  }
   static inline typename T::Builder init(PointerBuilder builder) {
     return typename T::Builder(builder.initStruct(structSize<T>()));
   }
@@ -78,6 +81,9 @@ struct PointerHelpers<List<T>, Kind::LIST> {
   static inline void set(PointerBuilder builder, typename List<T>::Reader value) {
     builder.setList(value.reader);
   }
+  static inline void setCanonical(PointerBuilder builder, typename List<T>::Reader value) {
+    builder.setList(value.reader, true);
+  }
   static void set(PointerBuilder builder, kj::ArrayPtr<const ReaderFor<T>> value) {
     auto l = init(builder, value.size());
     uint i = 0;
@@ -117,6 +123,9 @@ struct PointerHelpers<T, Kind::BLOB> {
   static inline void set(PointerBuilder builder, typename T::Reader value) {
     builder.setBlob<T>(value);
   }
+  static inline void setCanonical(PointerBuilder builder, typename T::Reader value) {
+    builder.setBlob<T>(value);
+  }
   static inline typename T::Builder init(PointerBuilder builder, uint size) {
     return builder.initBlob<T>(size * BYTES);
   }