Pull TextBlob out of capnpc-capnp and into KJ as kj::StringTree. Use it in the…

Pull TextBlob out of capnpc-capnp and into KJ as kj::StringTree.  Use it in the stringification code to make that code cleaner and avoid using iostreams.

c++/Makefile.am

@@ -114,6 +114,7 @@ includekj_HEADERS =                                            \
   src/kj/array.h                                               \
   src/kj/vector.h                                              \
   src/kj/string.h                                              \
+  src/kj/string-tree.h                                         \
   src/kj/exception.h                                           \
   src/kj/debug.h                                               \
   src/kj/arena.h                                               \
@@ -156,6 +157,7 @@ libcapnp_la_SOURCES=                                           \
   src/kj/memory.c++                                            \
   src/kj/array.c++                                             \
   src/kj/string.c++                                            \
+  src/kj/string-tree.c++                                       \
   src/kj/exception.c++                                         \
   src/kj/debug.c++                                             \
   src/kj/arena.c++                                             \
@@ -233,6 +235,7 @@ capnp_test_SOURCES =                                           \
   src/kj/memory-test.c++                                       \
   src/kj/array-test.c++                                        \
   src/kj/string-test.c++                                       \
+  src/kj/string-tree-test.c++                                  \
   src/kj/exception-test.c++                                    \
   src/kj/debug-test.c++                                        \
   src/kj/arena-test.c++                                        \

c++/src/capnp/compiler/capnp.c++

@@ -451,18 +451,14 @@ private:
     {
       ParseErrorCatcher catcher;
       if (pretty) {
-        text = prettyPrint(root);
+        text = kj::str(prettyPrint(root), '\n');
       } else {
-        text = kj::str(root);
+        text = kj::str(root, '\n');
       }
       exception = kj::mv(catcher.exception);
     }
 
-    kj::ArrayPtr<const byte> pieces[2];
-    pieces[0] = kj::arrayPtr(reinterpret_cast<const byte*>(text.begin()), text.size());
-    pieces[1] = kj::arrayPtr(reinterpret_cast<const byte*>("\n"), 1);
-
-    kj::FdOutputStream(STDOUT_FILENO).write(kj::arrayPtr(pieces, KJ_ARRAY_SIZE(pieces)));
+    kj::FdOutputStream(STDOUT_FILENO).write(text.begin(), text.size());
 
     KJ_IF_MAYBE(e, exception) {
       context.error(kj::str("*** error in previous message ***\n", *e, "\n*** end error ***"));

c++/src/capnp/compiler/lexer.c++

@@ -139,7 +139,7 @@ constexpr auto docComment = p::optional(p::sequence(
 }  // namespace
 
 Lexer::Lexer(Orphanage orphanageParam, const ErrorReporter& errorReporterParam)
-    : orphanage(orphanageParam), errorReporter(errorReporterParam) {
+    : orphanage(orphanageParam) {
 
   // Note that because passing an lvalue to a parser constructor uses it by-referencee, it's safe
   // for us to use parsers.tokenSequence even though we haven't yet constructed it.

c++/src/capnp/compiler/lexer.h

@@ -91,7 +91,6 @@ public:
 
 private:
   Orphanage orphanage;
-  const ErrorReporter& errorReporter;
   kj::Arena arena;
   Parsers parsers;
 };

c++/src/capnp/dynamic.h

@@ -201,7 +201,7 @@ private:
 
   friend struct DynamicStruct;
   friend class DynamicUnion::Builder;
-  friend kj::String _::unionString(
+  friend kj::StringTree _::unionString(
       _::StructReader reader, const _::RawSchema& schema, uint memberIndex);
 };
 
@@ -305,7 +305,7 @@ private:
   friend class MessageBuilder;
   template <typename T, ::capnp::Kind k>
   friend struct ::capnp::ToDynamic_;
-  friend kj::String _::structString(
+  friend kj::StringTree _::structString(
       _::StructReader reader, const _::RawSchema& schema);
   friend class Orphanage;
   friend class Orphan<DynamicStruct>;
@@ -666,16 +666,16 @@ private:
   // specialization.  Has a method apply() which does the work.
 };
 
-kj::String KJ_STRINGIFY(const DynamicValue::Reader& value);
-kj::String KJ_STRINGIFY(const DynamicValue::Builder& value);
-kj::String KJ_STRINGIFY(DynamicEnum value);
-kj::String KJ_STRINGIFY(const DynamicObject& value);
-kj::String KJ_STRINGIFY(const DynamicUnion::Reader& value);
-kj::String KJ_STRINGIFY(const DynamicUnion::Builder& value);
-kj::String KJ_STRINGIFY(const DynamicStruct::Reader& value);
-kj::String KJ_STRINGIFY(const DynamicStruct::Builder& value);
-kj::String KJ_STRINGIFY(const DynamicList::Reader& value);
-kj::String KJ_STRINGIFY(const DynamicList::Builder& value);
+kj::StringTree KJ_STRINGIFY(const DynamicValue::Reader& value);
+kj::StringTree KJ_STRINGIFY(const DynamicValue::Builder& value);
+kj::StringTree KJ_STRINGIFY(DynamicEnum value);
+kj::StringTree KJ_STRINGIFY(const DynamicObject& value);
+kj::StringTree KJ_STRINGIFY(const DynamicUnion::Reader& value);
+kj::StringTree KJ_STRINGIFY(const DynamicUnion::Builder& value);
+kj::StringTree KJ_STRINGIFY(const DynamicStruct::Reader& value);
+kj::StringTree KJ_STRINGIFY(const DynamicStruct::Builder& value);
+kj::StringTree KJ_STRINGIFY(const DynamicList::Reader& value);
+kj::StringTree KJ_STRINGIFY(const DynamicList::Builder& value);
 
 // -------------------------------------------------------------------
 // Orphan <-> Dynamic glue

c++/src/capnp/generated-header-support.h

@@ -30,6 +30,7 @@
 #include "list.h"
 #include "orphan.h"
 #include <kj/string.h>
+#include <kj/string-tree.h>
 
 namespace capnp {
 
@@ -216,18 +217,18 @@ struct UnionParentType_;
 template <typename T>
 using UnionParentType = typename UnionParentType_<T>::Type;
 
-kj::String structString(StructReader reader, const RawSchema& schema);
-kj::String unionString(StructReader reader, const RawSchema& schema, uint memberIndex);
+kj::StringTree structString(StructReader reader, const RawSchema& schema);
+kj::StringTree unionString(StructReader reader, const RawSchema& schema, uint memberIndex);
 // Declared here so that we can declare inline stringify methods on generated types.
 // Defined in stringify.c++, which depends on dynamic.c++, which is allowed not to be linked in.
 
 template <typename T>
-inline kj::String structString(StructReader reader) {
+inline kj::StringTree structString(StructReader reader) {
   return structString(reader, rawSchema<T>());
 }
 
 template <typename T>
-inline kj::String unionString(StructReader reader) {
+inline kj::StringTree unionString(StructReader reader) {
   return unionString(reader, rawSchema<UnionParentType<T>>(), unionMemberIndex<T>());
 }
 

c++/src/capnp/pretty-print.h

@@ -25,14 +25,14 @@
 #define CAPNP_PRETTY_PRINT_H_
 
 #include "dynamic.h"
-#include <kj/string.h>
+#include <kj/string-tree.h>
 
 namespace capnp {
 
-kj::String prettyPrint(DynamicStruct::Reader value);
-kj::String prettyPrint(DynamicStruct::Builder value);
-kj::String prettyPrint(DynamicList::Reader value);
-kj::String prettyPrint(DynamicList::Builder value);
+kj::StringTree prettyPrint(DynamicStruct::Reader value);
+kj::StringTree prettyPrint(DynamicStruct::Builder value);
+kj::StringTree prettyPrint(DynamicList::Reader value);
+kj::StringTree prettyPrint(DynamicList::Builder value);
 // Print the given Cap'n Proto struct or list with nice indentation.  Note that you can pass any
 // struct or list reader or builder type to this method, since they can be implicitly converted
 // to one of the dynamic types.

c++/src/capnp/schema.h

@@ -127,9 +127,9 @@ private:
     return StructSchema(&_::rawSchema<T>());
   }
   friend class Schema;
-  friend kj::String _::structString(
+  friend kj::StringTree _::structString(
       _::StructReader reader, const _::RawSchema& schema);
-  friend kj::String _::unionString(
+  friend kj::StringTree _::unionString(
       _::StructReader reader, const _::RawSchema& schema, uint memberIndex);
 };
 

c++/src/capnp/stringify-test.c++

@@ -147,7 +147,7 @@ TEST(Stringify, PrettyPrint) {
   MallocMessageBuilder builder;
   auto root = builder.initRoot<TestAllTypes>();
 
-  EXPECT_EQ("()", prettyPrint(root));
+  EXPECT_EQ("()", prettyPrint(root).flatten());
 
   initTestMessage(root);
 
@@ -181,7 +181,8 @@ TEST(Stringify, PrettyPrint) {
       "    dataField = \"qux\",\n"
       "    structField = (\n"
       "      textField = \"nested\",\n"
-      "      structField = (textField = \"really nested\")),\n"
+      "      structField = (\n"
+      "        textField = \"really nested\" ) ),\n"
       "    enumField = baz,\n"
       "    voidList = [void, void, void],\n"
       "    boolList = [false, true, false, true, true],\n"
@@ -199,10 +200,10 @@ TEST(Stringify, PrettyPrint) {
       "    textList = [\"quux\", \"corge\", \"grault\"],\n"
       "    dataList = [\"garply\", \"waldo\", \"fred\"],\n"
       "    structList = [\n"
-      "      (textField = \"x structlist 1\"),\n"
-      "      (textField = \"x structlist 2\"),\n"
-      "      (textField = \"x structlist 3\")],\n"
-      "    enumList = [qux, bar, grault]),\n"
+      "      ( textField = \"x structlist 1\" ),\n"
+      "      ( textField = \"x structlist 2\" ),\n"
+      "      ( textField = \"x structlist 3\" ) ],\n"
+      "    enumList = [qux, bar, grault] ),\n"
       "  enumField = corge,\n"
       "  voidList = [void, void, void, void, void, void],\n"
       "  boolList = [true, false, false, true],\n"
@@ -219,11 +220,11 @@ TEST(Stringify, PrettyPrint) {
       "  textList = [\"plugh\", \"xyzzy\", \"thud\"],\n"
       "  dataList = [\"oops\", \"exhausted\", \"rfc3092\"],\n"
       "  structList = [\n"
-      "    (textField = \"structlist 1\"),\n"
-      "    (textField = \"structlist 2\"),\n"
-      "    (textField = \"structlist 3\")],\n"
-      "  enumList = [foo, garply])",
-      prettyPrint(root));
+      "    ( textField = \"structlist 1\" ),\n"
+      "    ( textField = \"structlist 2\" ),\n"
+      "    ( textField = \"structlist 3\" ) ],\n"
+      "  enumList = [foo, garply] )",
+      prettyPrint(root).flatten());
 }
 
 TEST(Stringify, PrettyPrintAdvanced) {
@@ -241,58 +242,52 @@ TEST(Stringify, PrettyPrintAdvanced) {
     list[2].setTextField("baz");
 
     EXPECT_EQ(
-        "(structList = [\n"
-        "  ( int32Field = 123,\n"
-        "    textField = \"foo\"),\n"
-        "  ( int32Field = 456,\n"
-        "    textField = \"bar\"),\n"
-        "  ( int32Field = 789,\n"
-        "    textField = \"baz\")])",
-        prettyPrint(root));
+        "( structList = [\n"
+        "    (int32Field = 123, textField = \"foo\"),\n"
+        "    (int32Field = 456, textField = \"bar\"),\n"
+        "    (int32Field = 789, textField = \"baz\") ] )",
+        prettyPrint(root).flatten());
 
     root.setInt32Field(55);
 
     EXPECT_EQ(
         "( int32Field = 55,\n"
         "  structList = [\n"
-        "    ( int32Field = 123,\n"
-        "      textField = \"foo\"),\n"
-        "    ( int32Field = 456,\n"
-        "      textField = \"bar\"),\n"
-        "    ( int32Field = 789,\n"
-        "      textField = \"baz\")])",
-        prettyPrint(root));
+        "    (int32Field = 123, textField = \"foo\"),\n"
+        "    (int32Field = 456, textField = \"bar\"),\n"
+        "    (int32Field = 789, textField = \"baz\") ] )",
+        prettyPrint(root).flatten());
   }
 
   {
     auto root = builder.initRoot<test::TestLists>();
     auto ll = root.initInt32ListList(3);
-    ll.set(0, {123, 456, 789});
-    ll.set(1, {234, 567, 891});
-    ll.set(2, {345, 678, 912});
+    ll.set(0, {123, 456, 789, 1234567890});
+    ll.set(1, {234, 567, 891, 1234567890});
+    ll.set(2, {345, 678, 912, 1234567890});
 
     EXPECT_EQ(
-        "[ [123, 456, 789],\n"
-        "  [234, 567, 891],\n"
-        "  [345, 678, 912]]",
-        prettyPrint(ll));
+        "[ [123, 456, 789, 1234567890],\n"
+        "  [234, 567, 891, 1234567890],\n"
+        "  [345, 678, 912, 1234567890] ]",
+        prettyPrint(ll).flatten());
 
     EXPECT_EQ(
-        "(int32ListList = [\n"
-        "  [123, 456, 789],\n"
-        "  [234, 567, 891],\n"
-        "  [345, 678, 912]])",
-        prettyPrint(root));
+        "( int32ListList = [\n"
+        "    [123, 456, 789, 1234567890],\n"
+        "    [234, 567, 891, 1234567890],\n"
+        "    [345, 678, 912, 1234567890] ] )",
+        prettyPrint(root).flatten());
 
     root.initList8(0);
 
     EXPECT_EQ(
         "( list8 = [],\n"
         "  int32ListList = [\n"
-        "    [123, 456, 789],\n"
-        "    [234, 567, 891],\n"
-        "    [345, 678, 912]])",
-        prettyPrint(root));
+        "    [123, 456, 789, 1234567890],\n"
+        "    [234, 567, 891, 1234567890],\n"
+        "    [345, 678, 912, 1234567890] ] )",
+        prettyPrint(root).flatten());
 
     auto l8 = root.initList8(1);
     l8[0].setF(12);
@@ -300,24 +295,22 @@ TEST(Stringify, PrettyPrintAdvanced) {
     EXPECT_EQ(
         "( list8 = [(f = 12)],\n"
         "  int32ListList = [\n"
-        "    [123, 456, 789],\n"
-        "    [234, 567, 891],\n"
-        "    [345, 678, 912]])",
-        prettyPrint(root));
+        "    [123, 456, 789, 1234567890],\n"
+        "    [234, 567, 891, 1234567890],\n"
+        "    [345, 678, 912, 1234567890] ] )",
+        prettyPrint(root).flatten());
 
     l8 = root.initList8(2);
     l8[0].setF(12);
     l8[1].setF(34);
 
     EXPECT_EQ(
-        "( list8 = [\n"
-        "    (f = 12),\n"
-        "    (f = 34)],\n"
+        "( list8 = [(f = 12), (f = 34)],\n"
         "  int32ListList = [\n"
-        "    [123, 456, 789],\n"
-        "    [234, 567, 891],\n"
-        "    [345, 678, 912]])",
-        prettyPrint(root));
+        "    [123, 456, 789, 1234567890],\n"
+        "    [234, 567, 891, 1234567890],\n"
+        "    [345, 678, 912, 1234567890] ] )",
+        prettyPrint(root).flatten());
   }
 
   {
@@ -326,19 +319,21 @@ TEST(Stringify, PrettyPrintAdvanced) {
     auto s = root.getUn().initAllTypes();
     EXPECT_EQ(
         "(un = allTypes())",
-        prettyPrint(root));
+        prettyPrint(root).flatten());
 
     s.setInt32Field(123);
     EXPECT_EQ(
-        "(un = allTypes(int32Field = 123))",
-        prettyPrint(root));
+        "( un = allTypes(int32Field = 123) )",
+        prettyPrint(root).flatten());
 
     s.setTextField("foo");
+    s.setUInt64Field(0xffffffffffffffffull);
     EXPECT_EQ(
-        "(un = allTypes(\n"
-        "  int32Field = 123,\n"
-        "  textField = \"foo\"))",
-        prettyPrint(root));
+        "( un = allTypes(\n"
+        "    int32Field = 123,\n"
+        "    uInt64Field = 18446744073709551615,\n"
+        "    textField = \"foo\" ) )",
+        prettyPrint(root).flatten());
   }
 }
 

c++/src/kj/array-test.c++

@@ -329,5 +329,11 @@ TEST(Array, OwnConst) {
   EXPECT_EQ(456, ci2[1]);
 }
 
+TEST(Array, Map) {
+  StringPtr foo = "abcd";
+  Array<char> bar = KJ_MAP(foo, c) -> char { return c + 1; };
+  EXPECT_STREQ("bcde", str(bar).cStr());
+}
+
 }  // namespace
 }  // namespace kj

c++/src/kj/array.h

@@ -460,6 +460,36 @@ private:
   T content[fixedSize];
 };
 
+// =======================================================================================
+// KJ_MAP_ARRAY
+
+#define KJ_MAP(array, elementName) \
+  ::kj::_::Mapper<decltype(array)>(array) * [&](decltype(*(array).begin()) elementName)
+// Applies some function to every element of an array, returning an Array of the results,  with
+// nice syntax.  Example:
+//
+//     StringPtr foo = "abcd";
+//     Array<char> bar = KJ_MAP(foo, c) -> char { return c + 1; };
+//     KJ_ASSERT(str(bar) == "bcde");
+
+namespace _ {  // private
+
+template <typename T>
+struct Mapper {
+  T array;
+  Mapper(T array): array(kj::fwd<T>(array)) {}
+  template <typename Func>
+  auto operator*(Func&& func) -> Array<decltype(func(*array.begin()))> {
+    auto builder = heapArrayBuilder<decltype(func(*array.begin()))>(array.size());
+    for (auto iter = array.begin(); iter != array.end(); ++iter) {
+      builder.add(func(*iter));
+    }
+    return builder.finish();
+  }
+};
+
+}  // namespace _ (private)
+
 // =======================================================================================
 // Inline implementation details
 

c++/src/kj/string-tree-test.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "string-tree.h"
+#include <gtest/gtest.h>
+
+namespace kj {
+namespace _ {  // private
+namespace {
+
+TEST(StringTree, StrTree) {
+  EXPECT_EQ("foobar", strTree("foo", "bar").flatten());
+  EXPECT_EQ("1 2 3 4", strTree(1, " ", 2u, " ", 3l, " ", 4ll).flatten());
+  EXPECT_EQ("1.5 foo 1e15 bar -3", strTree(1.5f, " foo ", 1e15, " bar ", -3).flatten());
+  EXPECT_EQ("foo", strTree('f', 'o', 'o').flatten());
+
+  {
+    StringTree tree = strTree(strTree(str("foo"), str("bar")), "baz");
+    EXPECT_EQ("foobarbaz", tree.flatten());
+
+    uint pieceCount = 0;
+    tree.visit([&](ArrayPtr<const char> part) { ++pieceCount; EXPECT_EQ(3, part.size()); });
+    EXPECT_EQ(3, pieceCount);
+  }
+
+  EXPECT_EQ("<foobarbaz>", str('<', strTree(str("foo"), "bar", str("baz")), '>'));
+}
+
+TEST(StringTree, DelimitedArray) {
+  Array<StringTree> arr = heapArray<StringTree>(4);
+  arr[0] = strTree("foo");
+  arr[1] = strTree("bar");
+  arr[2] = strTree("baz");
+  arr[3] = strTree("qux");
+
+  EXPECT_EQ("foo, bar, baz, qux", StringTree(kj::mv(arr), ", ").flatten());
+}
+
+}  // namespace
+}  // namespace _ (private)
+}  // namespace kj

c++/src/kj/string-tree.c++

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "string-tree.h"
+
+namespace kj {
+
+StringTree::StringTree(Array<StringTree>&& pieces, StringPtr delim)
+    : size_(0),
+      branches(heapArray<Branch>(pieces.size())) {
+  if (pieces.size() > 0) {
+    if (pieces.size() > 1 && delim.size() > 0) {
+      text = heapString((pieces.size() - 1) * delim.size());
+      size_ = text.size();
+    }
+
+    branches[0].index = 0;
+    branches[0].content = kj::mv(pieces[0]);
+    size_ += pieces[0].size();
+
+    for (uint i = 1; i < pieces.size(); i++) {
+      if (delim.size() > 0) {
+        memcpy(text.begin() + (i - 1) * delim.size(), delim.begin(), delim.size());
+      }
+      branches[i].index = i * delim.size();
+      branches[i].content = kj::mv(pieces[i]);
+      size_ += pieces[i].size();
+    }
+  }
+}
+
+String StringTree::flatten() const {
+  String result = heapString(size());
+  flattenTo(result.begin());
+  return result;
+}
+
+void StringTree::flattenTo(char* __restrict__ target) const {
+  visit([&target](ArrayPtr<const char> text) {
+    memcpy(target, text.begin(), text.size());
+    target += text.size();
+  });
+}
+
+}  // namespace kj

c++/src/kj/string-tree.h

+// Copyright (c) 2013, Kenton Varda <temporal@gmail.com>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef KJ_STRING_TREE_H_
+#define KJ_STRING_TREE_H_
+
+#include "string.h"
+
+namespace kj {
+
+class StringTree {
+  // A long string, represented internally as a tree of strings.  This data structure is like a
+  // String, but optimized for concatenation and iteration at the expense of seek time.  The
+  // structure is intended to be used for building large text blobs from many small pieces, where
+  // repeatedly concatenating smaller strings into larger ones would waste copies.  This structure
+  // is NOT intended for use cases requiring random access or computing substrings.  For those,
+  // you should use a Rope, which is a much more complicated data structure.
+  //
+  // The proper way to construct a StringTree is via kj::strTree(...), which works just like
+  // kj::str(...) but returns a StringTree rather than a String.
+  //
+  // KJ_STRINGIFY() functions that construct large strings from many smaller strings are encouraged
+  // to return StringTree rather than a flat char container.
+
+public:
+  inline StringTree(): size_(0) {}
+  inline StringTree(String&& text): size_(text.size()), text(kj::mv(text)) {}
+
+  StringTree(Array<StringTree>&& pieces, StringPtr delim);
+  // Build a StringTree by concatenating the given pieces, delimited by the given delimiter
+  // (e.g. ", ").
+
+  inline size_t size() const { return size_; }
+
+  template <typename Func>
+  void visit(Func&& func) const;
+
+  String flatten() const;
+  // Return the contents as a string.
+
+  // TODO(someday):  flatten() when *this is an rvalue and when branches.size() == 0 could simply
+  //   return `kj::mv(text)`.  Requires reference qualifiers (Clang 3.3 / GCC 4.8).
+
+  void flattenTo(char* __restrict__ target) const;
+  // Copy the contents to the given character array.  Does not add a NUL terminator.
+
+private:
+  size_t size_;
+  String text;
+
+  struct Branch;
+  Array<Branch> branches;  // In order.
+
+  inline void fill(char* pos, size_t branchIndex);
+  template <typename First, typename... Rest>
+  void fill(char* pos, size_t branchIndex, First&& first, Rest&&... rest);
+  template <typename... Rest>
+  void fill(char* pos, size_t branchIndex, StringTree&& first, Rest&&... rest);
+  template <typename... Rest>
+  void fill(char* pos, size_t branchIndex, Array<char>&& first, Rest&&... rest);
+  template <typename... Rest>
+  void fill(char* pos, size_t branchIndex, String&& first, Rest&&... rest);
+
+  template <typename... Params>
+  static StringTree concat(Params&&... params);
+  static StringTree&& concat(StringTree&& param) { return kj::mv(param); }
+
+  template <typename T>
+  static inline size_t flatSize(const T& t) { return t.size(); }
+  static inline size_t flatSize(String&& s) { return 0; }
+  static inline size_t flatSize(StringTree&& s) { return 0; }
+
+  template <typename T>
+  static inline size_t branchCount(const T& t) { return 0; }
+  static inline size_t branchCount(String&& s) { return 1; }
+  static inline size_t branchCount(StringTree&& s) { return 1; }
+
+  template <typename... Params>
+  friend StringTree strTree(Params&&... params);
+};
+
+inline StringTree&& KJ_STRINGIFY(StringTree&& tree) { return kj::mv(tree); }
+inline const StringTree& KJ_STRINGIFY(const StringTree& tree) { return tree; }
+
+inline StringTree KJ_STRINGIFY(Array<StringTree>&& trees) { return StringTree(kj::mv(trees), ""); }
+
+template <typename... Params>
+StringTree strTree(Params&&... params);
+// Build a StringTree by stringifying the given parameters and concatenating the results.
+// If any of the parameters stringify to StringTree rvalues, they will be incorporated as
+// branches to avoid a copy.
+
+// =======================================================================================
+// Inline implementation details
+
+namespace _ {  // private
+
+template <typename... Rest>
+char* fill(char* __restrict__ target, const StringTree& first, Rest&&... rest) {
+  // Make str() work with stringifiers that return StringTree by patching fill().
+
+  first.flattenTo(target);
+  return fill(target + first.size(), kj::fwd<Rest>(rest)...);
+}
+
+template <typename T> constexpr bool isStringTree() { return false; }
+template <> constexpr bool isStringTree<StringTree>() { return true; }
+
+inline StringTree&& toStringTreeOrCharSequnece(StringTree&& tree) { return kj::mv(tree); }
+inline StringTree toStringTreeOrCharSequnece(String&& str) { return StringTree(kj::mv(str)); }
+
+template <typename T>
+inline auto toStringTreeOrCharSequnece(T&& value)
+    -> decltype(toCharSequence(kj::fwd<T>(value))) {
+  static_assert(!isStringTree<Decay<T>>(),
+      "When passing a StringTree into kj::strTree(), either pass it by rvalue "
+      "(use kj::mv(value)) or explicitly call value.flatten() to make a copy.");
+
+  return toCharSequence(kj::fwd<T>(value));
+}
+
+}  // namespace _ (private)
+
+struct StringTree::Branch {
+  size_t index;
+  // Index in `text` where this branch should be inserted.
+
+  StringTree content;
+};
+
+template <typename Func>
+void StringTree::visit(Func&& func) const {
+  size_t pos = 0;
+  for (auto& branch: branches) {
+    if (branch.index > pos) {
+      func(text.slice(pos, branch.index));
+      pos = branch.index;
+    }
+    branch.content.visit(func);
+  }
+  if (text.size() > pos) {
+    func(text.slice(pos, text.size()));
+  }
+}
+
+inline void StringTree::fill(char* pos, size_t branchIndex) {
+  KJ_IREQUIRE(pos == text.end() && branchIndex == branches.size(),
+              kj::str(text.end() - pos, ' ', branches.size() - branchIndex).cStr());
+}
+
+template <typename First, typename... Rest>
+void StringTree::fill(char* pos, size_t branchIndex, First&& first, Rest&&... rest) {
+  pos = _::fill(pos, kj::fwd<First>(first));
+  fill(pos, branchIndex, kj::fwd<Rest>(rest)...);
+}
+
+template <typename... Rest>
+void StringTree::fill(char* pos, size_t branchIndex, StringTree&& first, Rest&&... rest) {
+  branches[branchIndex].index = pos - text.begin();
+  branches[branchIndex].content = kj::mv(first);
+  fill(pos, branchIndex + 1, kj::fwd<Rest>(rest)...);
+}
+
+template <typename... Rest>
+void StringTree::fill(char* pos, size_t branchIndex, String&& first, Rest&&... rest) {
+  branches[branchIndex].index = pos - text.begin();
+  branches[branchIndex].content = StringTree(kj::mv(first));
+  fill(pos, branchIndex + 1, kj::fwd<Rest>(rest)...);
+}
+
+template <typename... Params>
+StringTree StringTree::concat(Params&&... params) {
+  StringTree result;
+  result.size_ = _::sum({params.size()...});
+  result.text = heapString(
+      _::sum({StringTree::flatSize(kj::fwd<Params>(params))...}));
+  result.branches = heapArray<StringTree::Branch>(
+      _::sum({StringTree::branchCount(kj::fwd<Params>(params))...}));
+  result.fill(result.text.begin(), 0, kj::fwd<Params>(params)...);
+  return result;
+}
+
+template <typename... Params>
+StringTree strTree(Params&&... params) {
+  return StringTree::concat(_::toStringTreeOrCharSequnece(kj::fwd<Params>(params))...);
+}
+
+}  // namespace kj
+
+#endif  // KJ_STRING_TREE_H_

c++/src/kj/string.h

@@ -33,6 +33,8 @@ namespace kj {
 class StringPtr;
 class String;
 
+class StringTree;   // string-tree.h
+
 // =======================================================================================
 // StringPtr -- A NUL-terminated ArrayPtr<const char> containing UTF-8 text.
 //
@@ -183,6 +185,12 @@ inline size_t sum(std::initializer_list<size_t> nums) {
 
 inline char* fill(char* ptr) { return ptr; }
 
+template <typename... Rest>
+char* fill(char* __restrict__ target, const StringTree& first, Rest&&... rest);
+// Make str() work with stringifiers that return StringTree by patching fill().
+//
+// Defined in string-tree.h.
+
 template <typename First, typename... Rest>
 char* fill(char* __restrict__ target, const First& first, Rest&&... rest) {
   auto i = first.begin();
@@ -250,9 +258,9 @@ struct Stringifier {
   CappedArray<char, sizeof(const void*) * 4> operator*(const void* s) const;
 
   template <typename T>
-  Array<char> operator*(ArrayPtr<T> arr) const;
+  String operator*(ArrayPtr<T> arr) const;
   template <typename T>
-  Array<char> operator*(const Array<T>& arr) const;
+  String operator*(const Array<T>& arr) const;
 };
 static constexpr Stringifier STR = Stringifier();
 
@@ -316,12 +324,12 @@ String strArray(T&& arr, const char* delim) {
 namespace _ {  // private
 
 template <typename T>
-inline Array<char> Stringifier::operator*(ArrayPtr<T> arr) const {
+inline String Stringifier::operator*(ArrayPtr<T> arr) const {
   return strArray(arr, ", ");
 }
 
 template <typename T>
-inline Array<char> Stringifier::operator*(const Array<T>& arr) const {
+inline String Stringifier::operator*(const Array<T>& arr) const {
   return strArray(arr, ", ");
 }
 

compiler/src/c++-header.mustache

@@ -193,10 +193,10 @@ private:
   friend struct ::capnp::List;
   friend class ::capnp::MessageBuilder;
   friend class ::capnp::Orphanage;
-  friend ::kj::String KJ_STRINGIFY({{typeFullName}}::Reader reader);
+  friend ::kj::StringTree KJ_STRINGIFY({{typeFullName}}::Reader reader);
 };
 
-inline ::kj::String KJ_STRINGIFY({{typeFullName}}::Reader reader) {
+inline ::kj::StringTree KJ_STRINGIFY({{typeFullName}}::Reader reader) {
 {{#typeStruct}}
   return ::capnp::_::structString<{{typeFullName}}>(reader._reader);
 {{/typeStruct}}
@@ -273,10 +273,10 @@ private:
   template <typename T, ::capnp::Kind k>
   friend struct ::capnp::ToDynamic_;
   friend class ::capnp::Orphanage;
-  friend ::kj::String KJ_STRINGIFY({{typeFullName}}::Builder builder);
+  friend ::kj::StringTree KJ_STRINGIFY({{typeFullName}}::Builder builder);
 };
 
-inline ::kj::String KJ_STRINGIFY({{typeFullName}}::Builder builder) {
+inline ::kj::StringTree KJ_STRINGIFY({{typeFullName}}::Builder builder) {
 {{#typeStruct}}
   return ::capnp::_::structString<{{typeFullName}}>(builder._builder.asReader());
 {{/typeStruct}}