Implemented InlineData, added a bunch of tests for list of lists and such, and fixed tons of bugs.

c++/src/capnproto/blob.h

@@ -40,6 +40,11 @@ struct Text {
   class Builder;
 };
 
+template <size_t size>
+struct InlineData: public Data {};
+// Alias for Data used specifically for InlineData fields.  This primarily exists so that
+// List<InlineData<n>> can be specialized.
+
 class Data::Reader {
   // Points to a blob of bytes.  The usual Reader rules apply -- Data::Reader behaves like a simple
   // pointer which does not own its target, can be passed by value, etc.

c++/src/capnproto/exception.c++

@@ -26,6 +26,8 @@
 #include "util.h"
 #include "logging.h"
 #include <unistd.h>
+#include <execinfo.h>
+#include <stdlib.h>
 
 namespace capnproto {
 
@@ -59,10 +61,15 @@ Exception::Exception(Nature nature, Durability durability, const char* file, int
       description(move(description)) {
   bool hasDescription = this->description != nullptr;
 
+  void* trace[16];
+  int traceCount = backtrace(trace, 16);
+  ArrayPtr<void*> traceArray = arrayPtr(trace, traceCount);
+
   // Must be careful to NUL-terminate this.
   whatStr = str(file, ":", line, ": ", nature,
                 durability == Durability::TEMPORARY ? " (temporary)" : "",
-                hasDescription ? ": " : "", this->description, '\0');
+                hasDescription ? ": " : "", this->description,
+                "\nstack: ", strArray(traceArray, " "), '\0');
 }
 
 Exception::Exception(const Exception& other) noexcept

c++/src/capnproto/layout.h

@@ -364,6 +364,14 @@ public:
                              const void* defaultValue, ByteCount defaultSize) const;
   // Same as *Text*, but for data blobs.
 
+  CAPNPROTO_ALWAYS_INLINE(Data::Builder getInlineDataField(
+      ByteCount offset, ByteCount size) const);
+  CAPNPROTO_ALWAYS_INLINE(void setInlineDataField(
+      ByteCount offset, ByteCount size, Data::Reader value) const);
+  CAPNPROTO_ALWAYS_INLINE(Data::Builder initInlineDataField(
+      ByteCount offset, ByteCount size) const);
+  // For InlineData.
+
   StructReader asReader() const;
   // Gets a StructReader pointing at the same memory.
 
@@ -445,6 +453,10 @@ public:
                             const void* defaultValue, ByteCount defaultSize) const;
   // Gets the data field, or the given default value if not initialized.
 
+  CAPNPROTO_ALWAYS_INLINE(Data::Reader getInlineDataField(
+      ByteCount offset, ByteCount size) const);
+  // Gets the inline data field.
+
   WireReferenceCount getReferenceCount() { return referenceCount; }
 
 private:
@@ -509,6 +521,10 @@ public:
   // Get the existing list element at the given index.  Returns an empty list if the element is
   // not initialized.
 
+  CAPNPROTO_ALWAYS_INLINE(ListBuilder slice(ElementCount start, ElementCount length) const);
+  // Get a list pointing at a slice of this list.  WARNING:  The second parameter is a length, not
+  // an end index, because this is what is most convenient at the only call site.
+
   Text::Builder initTextElement(ElementCount index, ByteCount size) const;
   // Initialize the text element to the given size in bytes (not including NUL terminator) and
   // return a Text::Builder which can be used to fill in the content.
@@ -522,6 +538,7 @@ public:
   Data::Builder initDataElement(ElementCount index, ByteCount size) const;
   void setDataElement(ElementCount index, Data::Reader value) const;
   Data::Builder getDataElement(ElementCount index) const;
+  // Like *Text*() but for Data.
 
   ListReader asReader(FieldSize elementSize) const;
   // Get a ListReader pointing at the same memory.  Use this version only for non-struct lists.
@@ -573,6 +590,10 @@ public:
   ListReader getListElement(ElementCount index, FieldSize expectedElementSize) const;
   // Get the list element at the given index.
 
+  CAPNPROTO_ALWAYS_INLINE(ListReader slice(ElementCount start, ElementCount length) const);
+  // Get a list pointing at a slice of this list.  WARNING:  The second parameter is a length, not
+  // an end index, because this is what is most convenient at the only call site.
+
   Text::Reader getTextElement(ElementCount index) const;
   // Get the text element.  If it is not initialized, returns an empty Text::Reader.
 
@@ -741,6 +762,22 @@ inline ListBuilder StructBuilder::getInlineStructListField(
       elementCount);
 }
 
+inline Data::Builder StructBuilder::getInlineDataField(
+    ByteCount offset, ByteCount size) const {
+  return Data::Builder(
+      reinterpret_cast<char*>(reinterpret_cast<byte*>(data) + offset), size / BYTES);
+}
+inline void StructBuilder::setInlineDataField(
+    ByteCount offset, ByteCount size, Data::Reader value) const {
+  getInlineDataField(offset, size).copyFrom(value);
+}
+inline Data::Builder StructBuilder::initInlineDataField(
+    ByteCount offset, ByteCount size) const {
+  byte* ptr = reinterpret_cast<byte*>(data) + offset;
+  memset(ptr, 0, size / BYTES);
+  return Data::Builder(reinterpret_cast<char*>(ptr), size / BYTES);
+}
+
 // -------------------------------------------------------------------
 
 template <typename T>
@@ -776,17 +813,20 @@ T StructReader::getDataField(ElementCount offset, typename MaskType<T>::Type mas
 inline StructReader StructReader::getInlineStructField(
     ByteCount dataOffset, ByteCount inlineDataSize,
     WireReferenceCount refIndex, WireReferenceCount inlineRefCount) const {
+  // TODO(soon):  Too complicated to be inlined?
   return StructReader(
       segment, reinterpret_cast<const byte*>(data) + dataOffset,
       // WireReference is incomplete here so we have to cast around...  Bah.
       reinterpret_cast<const WireReference*>(
           reinterpret_cast<const word*>(references) + refIndex * WORDS_PER_REFERENCE),
-      dataSize, inlineRefCount,
+      inlineDataSize * (dataOffset + inlineDataSize <= dataSize),
+      inlineRefCount * (refIndex + inlineRefCount <= referenceCount),
       nestingLimit);
 }
 
 inline ListReader StructReader::getInlineDataListField(
     ByteCount offset, ElementCount elementCount, FieldSize elementSize) const {
+  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
   return ListReader(
       segment, reinterpret_cast<const byte*>(data) + offset, nullptr,
       elementCount, bytesPerElement(elementSize), 0 * REFERENCES / ELEMENTS,
@@ -795,6 +835,7 @@ inline ListReader StructReader::getInlineDataListField(
 
 inline ListReader StructReader::getInlinePointerListField(
     WireReferenceCount offset, ElementCount elementCount) const {
+  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
   return ListReader(
       segment, nullptr,
       reinterpret_cast<const WireReference*>(
@@ -806,6 +847,7 @@ inline ListReader StructReader::getInlinePointerListField(
 inline ListReader StructReader::getInlineStructListField(
     ByteCount dataOffset, WireReferenceCount ptrOffset, ElementCount elementCount,
     StructSize elementSize) const {
+  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
   return ListReader(
       segment, reinterpret_cast<const byte*>(data) + dataOffset,
       reinterpret_cast<const WireReference*>(
@@ -814,6 +856,12 @@ inline ListReader StructReader::getInlineStructListField(
       elementSize.dataBytes, elementSize.pointers, nestingLimit);
 }
 
+inline Data::Reader StructReader::getInlineDataField(ByteCount offset, ByteCount size) const {
+  // TODO(soon):  Bounds check!  Needs to fall back to some common zero'd region.
+  return Data::Reader(reinterpret_cast<const char*>(reinterpret_cast<const byte*>(data) + offset),
+                      size / BYTES);
+}
+
 // -------------------------------------------------------------------
 
 inline ElementCount ListBuilder::size() { return elementCount; }
@@ -856,6 +904,14 @@ inline void ListBuilder::setDataElement<bool>(ElementCount index, bool value) co
 template <>
 inline void ListBuilder::setDataElement<Void>(ElementCount index, Void value) const {}
 
+inline ListBuilder ListBuilder::slice(ElementCount start, ElementCount length) const {
+  return ListBuilder(segment,
+      reinterpret_cast<byte*>(data) + start * stepBytes,
+      reinterpret_cast<WireReference*>(
+          reinterpret_cast<word*>(pointers) + start * stepPointers * WORDS_PER_REFERENCE),
+      stepBytes, stepPointers, length);
+}
+
 // -------------------------------------------------------------------
 
 inline ElementCount ListReader::size() { return elementCount; }
@@ -879,6 +935,14 @@ inline Void ListReader::getDataElement<Void>(ElementCount index) const {
   return Void::VOID;
 }
 
+inline ListReader ListReader::slice(ElementCount start, ElementCount length) const {
+  return ListReader(segment,
+      reinterpret_cast<const byte*>(data) + start * stepBytes,
+      reinterpret_cast<const WireReference*>(
+          reinterpret_cast<const word*>(pointers) + start * stepPointers * WORDS_PER_REFERENCE),
+      length, stepBytes, stepPointers, structDataSize, structReferenceCount, nestingLimit);
+}
+
 }  // namespace internal
 }  // namespace capnproto
 

c++/src/capnproto/logging-test.c++

@@ -45,13 +45,27 @@ public:
 
   void onRecoverableException(Exception&& exception) override {
     text += "recoverable exception: ";
-    text += exception.what();
+    // Only take the first line of "what" because the second line is a stack trace.
+    const char* what = exception.what();
+    const char* end = strchr(what, '\n');
+    if (end == nullptr) {
+      text += exception.what();
+    } else {
+      text.append(what, end);
+    }
     text += '\n';
   }
 
   void onFatalException(Exception&& exception) override {
     text += "fatal exception: ";
-    text += exception.what();
+    // Only take the first line of "what" because the second line is a stack trace.
+    const char* what = exception.what();
+    const char* end = strchr(what, '\n');
+    if (end == nullptr) {
+      text += exception.what();
+    } else {
+      text.append(what, end);
+    }
     text += '\n';
     throw MockException();
   }

c++/src/capnproto/test-util.h

@@ -48,30 +48,35 @@ inline std::ostream& operator<<(std::ostream& os, Void) {
 
 namespace internal {
 
-// Explicitly import each of these to make sure they're really located in capnproto::test and not,
-// say, the global namespace.
-using ::capnproto::test::TestAllTypes;
-using ::capnproto::test::TestDefaults;
-using ::capnproto::test::TestEnum;
-using ::capnproto::test::TestUnion;
-using ::capnproto::test::TestUnionDefaults;
-using ::capnproto::test::TestNestedTypes;
-using ::capnproto::test::TestUsing;
-using ::capnproto::test::TestInlineLayout;
-using ::capnproto::test::TestInlineUnions;
-using ::capnproto::test::TestInlineDefaults;
+namespace test = capnproto_test::capnproto::test;
 
-void initTestMessage(test::TestAllTypes::Builder builder);
-void initTestMessage(test::TestDefaults::Builder builder);
+// We don't use "using namespace" to pull these in because then things would still compile
+// correctly if they were generated in the global namespace.
+using ::capnproto_test::capnproto::test::TestAllTypes;
+using ::capnproto_test::capnproto::test::TestDefaults;
+using ::capnproto_test::capnproto::test::TestEnum;
+using ::capnproto_test::capnproto::test::TestUnion;
+using ::capnproto_test::capnproto::test::TestUnionDefaults;
+using ::capnproto_test::capnproto::test::TestNestedTypes;
+using ::capnproto_test::capnproto::test::TestUsing;
+using ::capnproto_test::capnproto::test::TestInlineLayout;
+using ::capnproto_test::capnproto::test::TestInlineUnions;
+using ::capnproto_test::capnproto::test::TestInlineDefaults;
 
-void checkTestMessage(test::TestAllTypes::Builder builder);
-void checkTestMessage(test::TestDefaults::Builder builder);
+void initTestMessage(TestAllTypes::Builder builder);
+void initTestMessage(TestDefaults::Builder builder);
+void initTestMessage(TestInlineDefaults::Builder builder);
 
-void checkTestMessage(test::TestAllTypes::Reader reader);
-void checkTestMessage(test::TestDefaults::Reader reader);
+void checkTestMessage(TestAllTypes::Builder builder);
+void checkTestMessage(TestDefaults::Builder builder);
+void checkTestMessage(TestInlineDefaults::Builder builder);
 
-void checkTestMessageAllZero(test::TestAllTypes::Builder builder);
-void checkTestMessageAllZero(test::TestAllTypes::Reader reader);
+void checkTestMessage(TestAllTypes::Reader reader);
+void checkTestMessage(TestDefaults::Reader reader);
+void checkTestMessage(TestInlineDefaults::Reader reader);
+
+void checkTestMessageAllZero(TestAllTypes::Builder builder);
+void checkTestMessageAllZero(TestAllTypes::Reader reader);
 
 }  // namespace internal
 }  // namespace capnproto

c++/src/capnproto/test.capnp

@@ -23,7 +23,9 @@
 
 using Cxx = import "c++.capnp";
 
-$Cxx.namespace("capnproto::test");
+# Use a namespace likely to cause trouble if the generated code doesn't use fully-qualified
+# names for stuff in the capnproto namespace.
+$Cxx.namespace("capnproto_test::capnproto::test");
 
 enum TestEnum {
   foo @0;
@@ -406,6 +408,8 @@ struct TestInlineLists {
   structList64p  @20 : InlineList(TestInline64p, 3);
   structList128p @21 : InlineList(TestInline128p, 4);
   structList192p @22 : InlineList(TestInline192p, 2);
+
+  data @23 :InlineData(5);
 }
 
 struct TestStructLists {
@@ -428,6 +432,22 @@ struct TestStructLists {
   listP  @6 :List(StructP);
 }
 
+struct TestListLists {
+  int32ListList @0 :List(List(Int32));
+  textListList @1 :List(List(Text));
+  structListList @2 :List(List(TestAllTypes));
+
+  int32InlineListList @3 :List(InlineList(Int32, 7));
+  textInlineListList @4 :List(InlineList(Text, 5));
+  structInlineListList @5 :List(InlineList(TestInline32p, 3));
+  inlineDataList @6 :List(InlineData(9));
+
+  int32InlineListListList @7 :List(List(InlineList(Int32, 2)));
+  textInlineListListList @8 :List(List(InlineList(Text, 5)));
+  structInlineListListList @9 :List(List(InlineList(TestInline32p, 3)));
+  inlineDataListList @10 :List(List(InlineData(3)));
+}
+
 struct TestInlineDefaults {
   normal @0 :TestInlineLayout = (
       f0 = (f = void),
@@ -497,7 +517,9 @@ struct TestInlineDefaults {
                         (p0 = "quxbaz", p1 = "quxqux", p2 = "quxcorge")],
       structList192p = [(f = (f2 = 123456789012345),
                          p0 = "corgebaz", p1 = "corgequx", p2 = "corgecorge"),
-                        (p0 = "graultbaz", p1 = "graultqux", p2 = "graultcorge")]);
+                        (p0 = "graultbaz", p1 = "graultqux", p2 = "graultcorge")],
+
+      data = "12345");
 
   structLists @3 :TestStructLists = (
       list0  = [(f = void), (f = void)],
@@ -507,4 +529,30 @@ struct TestInlineDefaults {
       list32 = [(f = 123456789), (f = 234567890)],
       list64 = [(f = 1234567890123456), (f = 2345678901234567)],
       listP  = [(f = "foo"), (f = "bar")]);
+
+  listLists @4 :TestListLists = (
+      int32ListList = [[1, 2, 3], [4, 5], [12341234]],
+      textListList = [["foo", "bar"], ["baz"], ["qux", "corge"]],
+      structListList = [[(int32Field = 123), (int32Field = 456)], [(int32Field = 789)]],
+
+      int32InlineListList = [[1, 2, 3, 4, 5, 6, 123456789], [987654321, 6, 5, 4, 3, 2, 1]],
+      textInlineListList = [["grault1", "grault2", "grault3", "grault4", "grault5"],
+                            ["garply1", "garply2", "garply3", "garply4", "garply5"],
+                            ["waldo1", "waldo2", "waldo3", "waldo4", "waldo5"]],
+      structInlineListList =
+          [[(f=(f1=123), p0="fred1"), (f=(f1=456), p0="fred2"), (f=(f1=789), p0="fred3")],
+           [(f=(f1=321), p0="plugh1"), (f=(f1=654), p0="plugh2"), (f=(f1=987), p0="plugh3")],
+           [(f=(f1=111), p0="thud1"), (f=(f1=222), p0="thud2"), (f=(f1=333), p0="thud3")]],
+      inlineDataList = ["123456789", "234567890", "345678901", "456789012", "567890123"],
+
+      int32InlineListListList = [[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10]], [[1234567,7654321]]],
+      textInlineListListList = [[["1", "2", "3", "4", "5"],
+                                 ["foo", "bar", "baz", "qux", "corge"]],
+                                [["z", "y", "x", "w", "v"]]],
+      structInlineListListList =
+          [[[(f=(f1=123), p0="fred1"), (f=(f1=456), p0="fred2"), (f=(f1=789), p0="fred3")],
+            [(f=(f1=321), p0="plugh1"), (f=(f1=654), p0="plugh2"), (f=(f1=987), p0="plugh3")]],
+           [[(f=(f1=111), p0="thud1"), (f=(f1=222), p0="thud2"), (f=(f1=333), p0="thud3")]]],
+
+      inlineDataListList = [["foo", "bar", "baz"], ["123", "234"]]);
 }

c++/src/capnproto/util.c++

@@ -47,6 +47,7 @@ STRINGIFY_INT(long, "%ld");
 STRINGIFY_INT(unsigned long, "%lu");
 STRINGIFY_INT(long long, "%lld");
 STRINGIFY_INT(unsigned long long, "%llu");
+STRINGIFY_INT(const void*, "%p");
 
 #undef STRINGIFY_INT
 

c++/src/capnproto/util.h

@@ -125,7 +125,7 @@ template <typename Element>
 Element* fill(Element* ptr) { return ptr; }
 
 template <typename Element, typename First, typename... Rest>
-Element* fill(Element* __restrict__ target, First& first, Rest&&... rest) {
+Element* fill(Element* __restrict__ target, const First& first, Rest&&... rest) {
   auto i = first.begin();
   auto end = first.end();
   while (i != end) {
@@ -134,16 +134,6 @@ Element* fill(Element* __restrict__ target, First& first, Rest&&... rest) {
   return fill(target, std::forward<Rest>(rest)...);
 }
 
-template <typename Element, typename First, typename... Rest>
-Element* fill(Element* __restrict__ target, First&& first, Rest&&... rest) {
-  auto i = first.begin();
-  auto end = first.end();
-  while (i != end) {
-    *target++ = std::move(*i++);
-  }
-  return fill(target, std::forward<Rest>(rest)...);
-}
-
 template <typename Element, typename... Params>
 Array<Element> concat(Params&&... params) {
   // Concatenate a bunch of containers into a single Array.  The containers can be anything that
@@ -196,6 +186,10 @@ struct Stringifier {
   CappedArray<char, sizeof(unsigned long long) * 4> operator*(unsigned long long i) const;
   CappedArray<char, 24> operator*(float f) const;
   CappedArray<char, 32> operator*(double f) const;
+  CappedArray<char, sizeof(const void*) * 4> operator*(const void* s) const;
+
+  template <typename T>
+  Array<char> operator*(ArrayPtr<T> arr) const;
 };
 static constexpr Stringifier STR;
 
@@ -210,6 +204,34 @@ Array<char> str(Params&&... params) {
   return concat<char>(STR * std::forward<Params>(params)...);
 }
 
+template <typename T>
+Array<char> strArray(ArrayPtr<T> arr, const char* delim) {
+  size_t delimLen = strlen(delim);
+  decltype(STR * arr[0]) pieces[arr.size()];
+  size_t size = 0;
+  for (size_t i = 0; i < arr.size(); i++) {
+    if (i > 0) size += delimLen;
+    pieces[i] = STR * arr[i];
+    size += pieces[i].size();
+  }
+
+  Array<char> result = newArray<char>(size);
+  char* pos = result.begin();
+  for (size_t i = 0; i < arr.size(); i++) {
+    if (i > 0) {
+      memcpy(pos, delim, delimLen);
+      pos += delimLen;
+    }
+    pos = fill(pos, pieces[i]);
+  }
+  return result;
+}
+
+template <typename T>
+inline Array<char> Stringifier::operator*(ArrayPtr<T> arr) const {
+  return strArray(arr, ", ");
+}
+
 }  // namespace capnproto
 
 #endif  // CAPNPROTO_UTIL_H_

compiler/src/Compiler.hs

@@ -164,6 +164,7 @@ builtinTypeMap = Map.fromList
      [("List", DescBuiltinList),
       ("Inline", DescBuiltinInline),
       ("InlineList", DescBuiltinInlineList),
+      ("InlineData", DescBuiltinInlineData),
       ("id", DescBuiltinId)])
 
 ------------------------------------------------------------------------------------------
@@ -255,6 +256,12 @@ compileValue pos (InlineListType t s) (ListFieldValue l) = do
         makeError pos $ printf "Fixed-size list must have exactly %d elements." s
     return $ ListDesc elements
 
+compileValue pos (InlineDataType s) (StringFieldValue x) = let
+    bytes = map (fromIntegral . fromEnum) x
+    in if List.genericLength bytes == s
+        then succeed $ DataDesc bytes
+        else makeError pos $ printf "Fixed-size data must have exactly %d bytes." s
+
 compileValue pos (BuiltinType BuiltinVoid) _ = makeError pos "Void fields cannot have values."
 compileValue pos (BuiltinType BuiltinBool) _ = makeExpectError pos "boolean"
 compileValue pos (BuiltinType BuiltinInt8) _ = makeExpectError pos "integer"
@@ -275,6 +282,7 @@ compileValue pos (StructType _) _ = makeExpectError pos "parenthesized list of f
 compileValue pos (InterfaceType _) _ = makeError pos "Interfaces can't have default values."
 compileValue pos (ListType _) _ = makeExpectError pos "list"
 compileValue pos (InlineListType _ _) _ = makeExpectError pos "list"
+compileValue pos (InlineDataType _) _ = makeExpectError pos "string"
 
 descAsType _ (DescEnum desc) = succeed (EnumType desc)
 descAsType _ (DescStruct desc) = succeed (StructType desc)
@@ -286,7 +294,9 @@ descAsType name DescBuiltinList = makeError (declNamePos name) message where
 descAsType name DescBuiltinInline = makeError (declNamePos name) message where
             message = printf "'Inline' requires exactly one type parameter." (declNameString name)
 descAsType name DescBuiltinInlineList = makeError (declNamePos name) message where
-            message = printf "'InlineList' requires exactly one type parameter." (declNameString name)
+            message = printf "'InlineList' requires exactly two type parameters." (declNameString name)
+descAsType name DescBuiltinInlineData = makeError (declNamePos name) message where
+            message = printf "'InlineData' requires exactly one type parameter." (declNameString name)
 descAsType name _ = makeError (declNamePos name) message where
             message = printf "'%s' is not a type." (declNameString name)
 
@@ -328,9 +338,15 @@ compileType scope (TypeExpression n params) = do
                                    (structName s)
                     InlineListType _ _ -> makeError (declNamePos n)
                         "InlineList of InlineList not currently supported."
+                    InlineDataType _ -> makeError (declNamePos n)
+                        "InlineList of InlineData not currently supported."
                     _ -> return $ InlineListType inner size
             _ -> makeError (declNamePos n)
                 "'InlineList' requires exactly two type parameters: a type and a size."
+        DescBuiltinInlineData -> case params of
+            [TypeParameterInteger size] -> return $ InlineDataType size
+            _ -> makeError (declNamePos n)
+                "'InlineData' requires exactly one type parameter: the byte size of the data."
         _ -> case params of
             [] -> descAsType n desc
             _ -> makeError (declNamePos n) $
@@ -881,6 +897,10 @@ compileDecl scope
                 recover () (case typeDesc of
                     InlineStructType _ ->
                         makeError defaultPos "Inline fields cannot have default values."
+                    InlineListType _ _ ->
+                        makeError defaultPos "Inline fields cannot have default values."
+                    InlineDataType _ ->
+                        makeError defaultPos "Inline fields cannot have default values."
                     _ -> return ())
                 return result
             Nothing -> return Nothing

compiler/src/CxxGenerator.hs

@@ -92,11 +92,16 @@ isPrimitive (InlineStructType _) = False
 isPrimitive (InterfaceType _) = False
 isPrimitive (ListType _) = False
 isPrimitive (InlineListType _ _) = False
+isPrimitive (InlineDataType _) = False
 
 isBlob (BuiltinType BuiltinText) = True
 isBlob (BuiltinType BuiltinData) = True
+isBlob (InlineDataType _) = True
 isBlob _ = False
 
+isInlineBlob (InlineDataType _) = True
+isInlineBlob _ = False
+
 isStruct (StructType _) = True
 isStruct (InlineStructType _) = True
 isStruct _ = False
@@ -116,6 +121,18 @@ isPrimitiveList (ListType t) = isPrimitive t
 isPrimitiveList (InlineListType t _) = isPrimitive t
 isPrimitiveList _ = False
 
+isPointerElement (InlineDataType _) = False
+isPointerElement t = not (isPrimitive t || isStruct t || isInlineList t)
+
+isPointerList (ListType t) = isPointerElement t
+isPointerList (InlineListType t _) = isPointerElement t
+isPointerList _ = False
+
+isInlineBlobList (ListType t) = isInlineBlob t
+isInlineBlobList _ = False
+
+isStructList (ListType t@(InlineListType _ _)) = isStructList t
+isStructList (InlineListType t@(InlineListType _ _) _) = isStructList t
 isStructList (ListType t) = isStruct t
 isStructList (InlineListType t _) = isStruct t
 isStructList _ = False
@@ -125,8 +142,20 @@ isInlineList _ = False
 
 blobTypeString (BuiltinType BuiltinText) = "Text"
 blobTypeString (BuiltinType BuiltinData) = "Data"
+blobTypeString (InlineDataType _) = "Data"
+blobTypeString (ListType t) = blobTypeString t
+blobTypeString (InlineListType t _) = blobTypeString t
 blobTypeString _ = error "Not a blob."
 
+inlineMultiplier (InlineListType t s) = s * inlineMultiplier t
+inlineMultiplier (InlineDataType s) = s
+inlineMultiplier _ = 1
+
+listInlineMultiplierString (ListType t) = case inlineMultiplier t of
+    1 -> ""
+    s -> " * " ++ show s
+listInlineMultiplierString _ = error "Not a list."
+
 cxxTypeString (BuiltinType BuiltinVoid) = " ::capnproto::Void"
 cxxTypeString (BuiltinType BuiltinBool) = "bool"
 cxxTypeString (BuiltinType BuiltinInt8) = " ::int8_t"
@@ -146,7 +175,10 @@ cxxTypeString (StructType desc) = globalName $ DescStruct desc
 cxxTypeString (InlineStructType desc) = globalName $ DescStruct desc
 cxxTypeString (InterfaceType desc) = globalName $ DescInterface desc
 cxxTypeString (ListType t) = concat [" ::capnproto::List<", cxxTypeString t, ">"]
-cxxTypeString (InlineListType t _) = concat [" ::capnproto::List<", cxxTypeString t, ">"]
+cxxTypeString (InlineListType t s) =
+    concat [" ::capnproto::InlineList<", cxxTypeString t, ", ", show s, ">"]
+cxxTypeString (InlineDataType s) =
+    concat [" ::capnproto::InlineData<", show s, ">"]
 
 cxxFieldSizeString SizeVoid = "VOID";
 cxxFieldSizeString (SizeData Size1) = "BIT";
@@ -214,6 +246,10 @@ elementType (ListType t) = t
 elementType (InlineListType t _) = t
 elementType _ = error "Called elementType on non-list."
 
+inlineElementType (ListType t@(InlineListType _ _)) = inlineElementType t
+inlineElementType (InlineListType t@(InlineListType _ _) _) = inlineElementType t
+inlineElementType t = elementType t
+
 repeatedlyTake _ [] = []
 repeatedlyTake n l = take n l : repeatedlyTake n (drop n l)
 
@@ -247,11 +283,14 @@ fieldContext parent desc = mkStrContext context where
     context "fieldUpperCase" = MuVariable $ toUpperCaseWithUnderscores $ fieldName desc
     context "fieldIsPrimitive" = MuBool $ isPrimitive $ fieldType desc
     context "fieldIsBlob" = MuBool $ isBlob $ fieldType desc
+    context "fieldIsInlineBlob" = MuBool $ isInlineBlob $ fieldType desc
     context "fieldIsStruct" = MuBool $ isStruct $ fieldType desc
     context "fieldIsInlineStruct" = MuBool $ isInlineStruct $ fieldType desc
     context "fieldIsList" = MuBool $ isList $ fieldType desc
     context "fieldIsNonStructList" = MuBool $ isNonStructList $ fieldType desc
     context "fieldIsPrimitiveList" = MuBool $ isPrimitiveList $ fieldType desc
+    context "fieldIsPointerList" = MuBool $ isPointerList $ fieldType desc
+    context "fieldIsInlineBlobList" = MuBool $ isInlineBlobList $ fieldType desc
     context "fieldIsStructList" = MuBool $ isStructList $ fieldType desc
     context "fieldIsInlineList" = MuBool $ isInlineList $ fieldType desc
     context "fieldDefaultBytes" =
@@ -263,6 +302,7 @@ fieldContext parent desc = mkStrContext context where
     context "fieldOffset" = MuVariable $ fieldOffsetInteger $ fieldOffset desc
     context "fieldInlineListSize" = case fieldType desc of
         InlineListType _ n -> MuVariable n
+        InlineDataType n -> MuVariable n
         _ -> muNull
     context "fieldInlineDataOffset" = case fieldOffset desc of
         InlineCompositeOffset off _ size _ ->
@@ -278,13 +318,16 @@ fieldContext parent desc = mkStrContext context where
     context "fieldInlinePointerSize" = case fieldOffset desc of
         InlineCompositeOffset _ _ _ size -> MuVariable size
         _ -> muNull
+    context "fieldInlineMultiplier" = MuVariable $ listInlineMultiplierString $ fieldType desc
     context "fieldDefaultMask" = case fieldDefaultValue desc of
         Nothing -> MuVariable ""
         Just v -> MuVariable (if isDefaultZero v then "" else ", " ++ defaultMask v)
     context "fieldElementSize" =
-        MuVariable $ cxxFieldSizeString $ fieldSize $ elementType $ fieldType desc
+        MuVariable $ cxxFieldSizeString $ fieldSize $ inlineElementType $ fieldType desc
     context "fieldElementType" =
         MuVariable $ cxxTypeString $ elementType $ fieldType desc
+    context "fieldInlineElementType" =
+        MuVariable $ cxxTypeString $ inlineElementType $ fieldType desc
     context "fieldUnion" = case fieldUnion desc of
         Just (u, _) -> muJust $ unionContext context u
         Nothing -> muNull

compiler/src/Semantics.hs

@@ -57,6 +57,7 @@ data Desc = DescFile FileDesc
           | DescBuiltinList
           | DescBuiltinInline
           | DescBuiltinInlineList
+          | DescBuiltinInlineData
           | DescBuiltinId
 
 descName (DescFile      _) = "(top-level)"
@@ -75,6 +76,7 @@ descName (DescBuiltinType d) = builtinTypeName d
 descName DescBuiltinList = "List"
 descName DescBuiltinInline = "Inline"
 descName DescBuiltinInlineList = "InlineList"
+descName DescBuiltinInlineData = "InlineData"
 descName DescBuiltinId = "id"
 
 descId (DescFile      d) = fileId d
@@ -93,6 +95,7 @@ descId (DescBuiltinType _) = Nothing
 descId DescBuiltinList = Nothing
 descId DescBuiltinInline = Nothing
 descId DescBuiltinInlineList = Nothing
+descId DescBuiltinInlineData = Nothing
 descId DescBuiltinId = Just "0U0T3e_SnatEfk6UcH2tcjTt1E0"
 
 -- Gets the ID if explicitly defined, or generates it by appending ".name" to the parent's ID.
@@ -119,6 +122,7 @@ descParent (DescBuiltinType _) = error "Builtin type has no parent."
 descParent DescBuiltinList = error "Builtin type has no parent."
 descParent DescBuiltinInline = error "Builtin type has no parent."
 descParent DescBuiltinInlineList = error "Builtin type has no parent."
+descParent DescBuiltinInlineData = error "Builtin type has no parent."
 descParent DescBuiltinId = error "Builtin annotation has no parent."
 
 descFile (DescFile d) = d
@@ -140,6 +144,7 @@ descAnnotations (DescBuiltinType _) = Map.empty
 descAnnotations DescBuiltinList = Map.empty
 descAnnotations DescBuiltinInline = Map.empty
 descAnnotations DescBuiltinInlineList = Map.empty
+descAnnotations DescBuiltinInlineData = Map.empty
 descAnnotations DescBuiltinId = Map.empty
 
 descRuntimeImports (DescFile      _) = error "Not to be called on files."
@@ -158,6 +163,7 @@ descRuntimeImports (DescBuiltinType _) = []
 descRuntimeImports DescBuiltinList = []
 descRuntimeImports DescBuiltinInline = []
 descRuntimeImports DescBuiltinInlineList = []
+descRuntimeImports DescBuiltinInlineData = []
 descRuntimeImports DescBuiltinId = []
 
 type MemberMap = Map.Map String (Maybe Desc)
@@ -228,6 +234,7 @@ data TypeDesc = BuiltinType BuiltinType
               | InterfaceType InterfaceDesc
               | ListType TypeDesc
               | InlineListType TypeDesc Integer
+              | InlineDataType Integer
 
 typeRuntimeImports (BuiltinType _) = []
 typeRuntimeImports (EnumType d) = [descFile (DescEnum d)]
@@ -236,6 +243,7 @@ typeRuntimeImports (InlineStructType d) = [descFile (DescStruct d)]
 typeRuntimeImports (InterfaceType d) = [descFile (DescInterface d)]
 typeRuntimeImports (ListType d) = typeRuntimeImports d
 typeRuntimeImports (InlineListType d _) = typeRuntimeImports d
+typeRuntimeImports (InlineDataType _) = []
 
 data DataSectionSize = DataSection1 | DataSection8 | DataSection16 | DataSection32
                      | DataSectionWords Integer
@@ -337,6 +345,12 @@ fieldSize (InlineListType element size) = let
         SizeReference -> size
         SizeInlineComposite _ pc -> pc * size
     in SizeInlineComposite dataSection pointerCount
+fieldSize (InlineDataType size)
+    | size <= 0 = SizeInlineComposite (DataSectionWords 0) 0
+    | size <= 1 = SizeInlineComposite DataSection8 0
+    | size <= 2 = SizeInlineComposite DataSection16 0
+    | size <= 4 = SizeInlineComposite DataSection32 0
+    | otherwise = SizeInlineComposite (DataSectionWords (div (size + 7) 8)) 0
 
 -- Render the type descriptor's name as a string, appropriate for use in the given scope.
 typeName :: Desc -> TypeDesc -> String
@@ -347,6 +361,7 @@ typeName scope (InlineStructType desc) = descQualifiedName scope (DescStruct des
 typeName scope (InterfaceType desc) = descQualifiedName scope (DescInterface desc)
 typeName scope (ListType t) = "List(" ++ typeName scope t ++ ")"
 typeName scope (InlineListType t s) = printf "InlineList(%s, %d)" (typeName scope t) s
+typeName scope (InlineDataType s) = printf "InlineData(%d)" s
 
 -- Computes the qualified name for the given descriptor within the given scope.
 -- At present the scope is only used to determine whether the target is in the same file.  If
@@ -605,6 +620,7 @@ descToCode _ (DescBuiltinType _) = error "Can't print code for builtin type."
 descToCode _ DescBuiltinList = error "Can't print code for builtin type."
 descToCode _ DescBuiltinInline = error "Can't print code for builtin type."
 descToCode _ DescBuiltinInlineList = error "Can't print code for builtin type."
+descToCode _ DescBuiltinInlineData = error "Can't print code for builtin type."
 descToCode _ DescBuiltinId = error "Can't print code for builtin annotation."
 
 maybeBlockCode :: String -> [Desc] -> String

compiler/src/WireFormat.hs

@@ -88,6 +88,8 @@ encodePointerValue (InlineStructType _) _ =
 encodePointerValue (ListType elementType) (ListDesc items) = encodeList elementType items
 encodePointerValue (InlineListType _ _) _ =
     error "Tried to encode inline list as a pointer."
+encodePointerValue (InlineDataType _) _ =
+    error "Tried to encode inline data as a pointer."
 encodePointerValue _ _ = error "Unknown pointer type."
 
 -- Given a sorted list of (bitOffset, data), pack into a byte array.
@@ -191,6 +193,7 @@ structDataSectionValues assignments = let
         (pos, v2) <- case (t, v) of
             (InlineStructType _, StructValueDesc v2) -> structDataSectionValues v2
             (InlineListType t2 _, ListDesc v2) -> inlineListDataSectionValues t2 v2
+            (InlineDataType _, DataDesc v2) -> [(0, EncodedBytes v2)]
             _ -> error "Non-inline-composite had inline-composite offset."
         return (pos + bitOffset, v2)
 
@@ -211,6 +214,7 @@ structPointerSectionValues assignments = let
         (pos, v2) <- case (t, v) of
             (InlineStructType _, StructValueDesc v2) -> structPointerSectionValues v2
             (InlineListType t2 _, ListDesc v2) -> inlineListPointerSectionValues t2 v2
+            (InlineDataType _, DataDesc _) -> []
             _ -> error "Non-inline-composite had inline-composite offset."
         return (pos + off, v2)
 
@@ -261,29 +265,21 @@ encodeList (StructType desc) elements = let
                             (genericLength elements),
         concat [tag, elemBytes, childBytes])
 
--- A list of inline structs is encoded into separate data and pointer sections, and the
--- pointer to it is actually a struct pointer.
-encodeList (InlineStructType desc) elements = let
-    dataBytes = inlineStructListDataSection desc elements
-    (refBytes, childBytes) = inlineStructListPointerSection desc elements
-
-    in case (structDataSize desc, structPointerCount desc) of
-        (ds, 0) ->  -- If only data, encode as a primitive list.
-            (encodeListReference (SizeData $ dataSectionAlignment ds)
-                (div (genericLength elements * dataSectionBits ds)
-                     (dataSizeInBits (dataSectionAlignment ds))),
-             dataBytes)
-        (DataSectionWords 0, pc) ->  -- If only pointers, encode as a pointer list.
-            (encodeListReference SizeReference (genericLength elements * pc),
-             refBytes ++ childBytes)
-        (ds, pc) ->  -- Otherwise, encode as a struct.
-            (encodeInlineStructListReference ds pc (genericLength elements),
-             concat [dataBytes, refBytes, childBytes])
+encodeList (InlineStructType _) _ = error "Not supported:  List of inline structs."
 
 -- Encode a list of inline lists by just concatenating all the elements.  The number of inner
 -- lists can be determined at runtime by dividing the total size by the fixed inline list size.
+-- Note that this means if you have something like List(InlineList(UInt8, 3)) and the list has
+-- two elements, the total size will be 6 bytes -- we don't round the individual sub-lists up
+-- to power-of-two boundaries.
+encodeList (InlineListType (InlineStructType t) _) elements =
+    encodeList (StructType t) (concat [l | ListDesc l <- elements])
 encodeList (InlineListType t _) elements = encodeList t (concat [l | ListDesc l <- elements])
 
+-- Encode a list of inline data.  Similar deal to above.
+encodeList (InlineDataType _) elements =
+    encodePointerValue (BuiltinType BuiltinData) (DataDesc $ concat [l | DataDesc l <- elements])
+
 -- Encode primitive types.
 encodeList elementType elements = let
     eSize = fieldSize elementType
@@ -306,6 +302,7 @@ inlineListDataSectionValues elementType elements = case fieldSize elementType of
     (SizeInlineComposite _ _) -> case elementType of
         InlineStructType desc -> inlineStructListDataSectionValues desc elements
         InlineListType t _ -> inlineListDataSectionValues t (concat [l | ListDesc l <- elements])
+        InlineDataType _ -> [(0, EncodedBytes $ concat [l | DataDesc l <- elements])]
         _ -> error "Unknown inline composite type."
     SizeReference -> []
     SizeData size -> let
@@ -317,6 +314,7 @@ inlineListPointerSectionValues elementType elements = case fieldSize elementType
     (SizeInlineComposite _ _) -> case elementType of
         InlineStructType desc -> inlineStructListPointerSectionValues desc elements
         InlineListType t _ -> inlineListPointerSectionValues t (concat [l | ListDesc l <- elements])
+        InlineDataType _ -> []
         _ -> error "Unknown inline composite type."
     SizeReference -> zip [0..] $ map (encodePointerValue elementType) elements
     SizeData _ -> []
@@ -348,9 +346,7 @@ inlineStructListPointerSectionValues elementDesc elements = do
 encodeMessage (StructType desc) (StructValueDesc assignments) = let
     (dataBytes, refBytes, childBytes) = encodeStruct desc assignments 0
     in concat [encodeStructReference desc (0::Integer), dataBytes, refBytes, childBytes]
-encodeMessage (InlineStructType desc) val = encodeMessage (StructType desc) val
 encodeMessage (ListType elementType) (ListDesc elements) = let
     (ptr, listBytes) = encodeList elementType elements
     in ptr (0::Integer) ++ listBytes
-encodeMessage (InlineListType elementType _) val = encodeMessage (ListType elementType) val
 encodeMessage _ _ = error "Not a message."